Only run python 3 fixers if profile version < 19.05

or python_template_version < 3.

lib/galaxy/tools/__init__.py

@@ -464,6 +464,7 @@ def __init__(self, config_file, tool_source, app, guid=None, repository_id=None,
         self.guid = guid
         self.old_id = None
         self.version = None
+        self.python_template_version = None
         self._lineage = None
         self.dependencies = []
         # populate toolshed repository info, if available
@@ -643,6 +644,14 @@ def parse(self, tool_source, guid=None, dynamic=False):
             message = template % (self.id, self.profile)
             raise Exception(message)
 
+        self.python_template_version = tool_source.parse_python_template_version()
+        if self.python_template_version is None:
+            # If python_template_version not specified we assume tools with profile versions >= 19.05 are python 3 ready
+            if self.profile >= 19.05:
+                self.python_template_version = 3.5
+            else:
+                self.python_template_version = 2.7
+
         # Get the (user visible) name of the tool
         self.name = tool_source.parse_name()
         if not self.name and dynamic:

lib/galaxy/tools/actions/__init__.py

@@ -378,7 +378,8 @@ def handle_output(name, output, hidden=None):
                     wrapped_params.params,
                     inp_data,
                     inp_dataset_collections,
-                    input_ext
+                    input_ext,
+                    python_template_version=tool.python_template_version,
                 )
                 create_datasets = True
                 dataset = None
@@ -745,7 +746,7 @@ def get_output_name(self, output, dataset, tool, on_text, trans, incoming, histo
         if output.label:
             params['tool'] = tool
             params['on_string'] = on_text
-            return fill_template(output.label, context=params)
+            return fill_template(output.label, context=params, python_template_version=tool.python_template_version)
         else:
             return self._get_default_data_name(dataset, tool, on_text=on_text, trans=trans, incoming=incoming, history=history, params=params, job_params=job_params)
 
@@ -762,7 +763,7 @@ def set_metadata_defaults(self, output, dataset, tool, on_text, trans, incoming,
         if output.actions:
             for action in output.actions.actions:
                 if action.tag == "metadata" and action.default:
-                    metadata_new_value = fill_template(action.default, context=params).split(",")
+                    metadata_new_value = fill_template(action.default, context=params, python_template_version=tool.python_template_version).split(",")
                     dataset.metadata.__setattr__(str(action.name), metadata_new_value)
 
     def _get_default_data_name(self, dataset, tool, on_text=None, trans=None, incoming=None, history=None, params=None, job_params=None, **kwd):
@@ -925,7 +926,7 @@ def get_ext_or_implicit_ext(hda):
     return hda.ext
 
 
-def determine_output_format(output, parameter_context, input_datasets, input_dataset_collections, random_input_ext):
+def determine_output_format(output, parameter_context, input_datasets, input_dataset_collections, random_input_ext, python_template_version=3):
     """ Determines the output format for a dataset based on an abstract
     description of the output (galaxy.tools.parser.ToolOutput), the parameter
     wrappers, a map of the input datasets (name => HDA), and the last input
@@ -987,7 +988,7 @@ def determine_output_format(output, parameter_context, input_datasets, input_dat
                         if '$' not in check:
                             # allow a simple name or more complex specifications
                             check = '${%s}' % check
-                        if str(fill_template(check, context=parameter_context)) == when_elem.get('value', None):
+                        if fill_template(check, context=parameter_context, python_template_version=python_template_version) == when_elem.get('value', None):
                             ext = when_elem.get('format', ext)
                     except Exception:  # bad tag input value; possibly referencing a param within a different conditional when block or other nonexistent grouping construct
                         continue

lib/galaxy/tools/evaluation.py

@@ -469,7 +469,7 @@ def __build_command_line(self):
             return
         try:
             # Substituting parameters into the command
-            command_line = fill_template(command, context=param_dict)
+            command_line = fill_template(command, context=param_dict, python_template_version=self.tool.python_template_version)
             cleaned_command_line = []
             # Remove leading and trailing whitespace from each line for readability.
             for line in command_line.split('\n'):
@@ -575,7 +575,7 @@ def __build_config_file_text(self, content):
 
     def __write_workdir_file(self, config_filename, content, context, is_template=True):
         if is_template:
-            value = fill_template(content, context=context)
+            value = fill_template(content, context=context, python_template_version=self.tool.python_template_version)
         else:
             value = unicodify(content)
         with io.open(config_filename, "w", encoding='utf-8') as f:

lib/galaxy/tools/parser/cwl.py

@@ -150,6 +150,9 @@ def parse_requirements_and_containers(self):
     def parse_profile(self):
         return "16.04"
 
+    def parse_python_template_version(self):
+        return 3.5
+
 
 class CwlPageSource(PageSource):
 

lib/galaxy/tools/parser/interface.py

@@ -218,6 +218,12 @@ def parse_profile(self):
         """ Return tool profile version as Galaxy major e.g. 16.01 or 16.04.
         """
 
+    @abstractmethod
+    def parse_python_template_version(self):
+        """
+        Return minimum python version that the tool template has been developed against.
+        """
+
     def macro_paths(self):
         return []
 

lib/galaxy/tools/parser/xml.py

@@ -483,6 +483,12 @@ def parse_profile(self):
         # - Enable buggy interpreter attribute.
         return self.root.get("profile", "16.01")
 
+    def parse_python_template_version(self):
+        python_template_version = self.root.get("python_template_version", None)
+        if python_template_version is not None:
+            python_template_version = float(python_template_version)
+        return python_template_version
+
 
 def _test_elem_to_dict(test_elem, i):
     rval = dict(

lib/galaxy/tools/parser/yaml.py

@@ -162,6 +162,12 @@ def parse_tests_to_dict(self):
     def parse_profile(self):
         return self.root_dict.get("profile", "16.04")
 
+    def parse_python_template_version(self):
+        python_template_version = self.root_dict.get("python_template_version", None)
+        if python_template_version is not None:
+            python_template_version = float(python_template_version)
+        return python_template_version
+
 
 def _parse_test(i, test_dict):
     inputs = test_dict["inputs"]

lib/galaxy/tools/xsd/galaxy.xsd

@@ -175,13 +175,22 @@ set this to have values such as ``data_source`` or ``manage_data``.</xs:document
       </xs:attribute>
       <xs:attribute name="profile" type="xs:string">
         <xs:annotation>
-          <xs:documentation xml:lang="en">This string specified the minimum Galaxy
+          <xs:documentation xml:lang="en">This string specifies the minimum Galaxy
 version that should be required to run this tool. Certain legacy behaviors such
 as using standard error content to detect errors instead of exit code are disabled
 automatically if profile is set to any version newer than ``16.01``, such as
 ``16.04``.</xs:documentation>
         </xs:annotation>
       </xs:attribute>
+      <xs:attribute name="python_template_version" type="xs:float">
+        <xs:annotation>
+          <xs:documentation xml:lang="en">This string specifies the minimum Python
+version that is able to fill the Cheetah sections of the tool. If unset defaults
+to 2.7 if the profile is older than 19.01, otherwise defaults to 3.5. Galaxy will 
+attempt to convert Python statements in Cheetah sections using [future](http://python-future.org/)
+if Galaxy is run on Python 3 and ``python_template_version`` is below 3.</xs:documentation>
+        </xs:annotation>
+      </xs:attribute>
       <xs:attribute name="workflow_compatible" type="xs:boolean" default="true">
         <xs:annotation>
           <xs:documentation xml:lang="en">This attribute indicates if

lib/galaxy/util/template.py

@@ -38,6 +38,7 @@ def fill_template(template_text,
                   compiler_class=Compiler,
                   first_exception=None,
                   futurized=False,
+                  python_template_version=3,
                   **kwargs):
     """Fill a cheetah template out for specified context.
 
@@ -56,13 +57,15 @@ def fill_template(template_text,
     except NotFound as e:
         if first_exception is None:
             first_exception = e
-        if retry > 0 and sys.version_info.major > 2:
+        if python_template_version < 3 and retry > 0 and sys.version_info.major > 2:
             tb = e.__traceback__
             last_stack = traceback.extract_tb(tb)[-1]
             if last_stack.name == '<listcomp>':
-                # On python 3 list,dict and set comprehensions as well as generator expressions
+                # On python 3 list, dict and set comprehensions as well as generator expressions
                 # have their own local scope, which prevents accessing frame variables in cheetah.
-                # We can work around this by replacing `$var` with `var`
+                # We can work around this by replacing `$var` with `var`, but we only do this for
+                # list comprehensions, as this has never worked for dict or set comprehensions or
+                # generator expressions in Cheetah.
                 var_not_found = e.args[0].split("'")[1]
                 replace_str = 'VFFSL(SL,"%s",True)' % var_not_found
                 lineno = last_stack.lineno - 1
@@ -80,7 +83,7 @@ def fill_template(template_text,
     except Exception as e:
         if first_exception is None:
             first_exception = e
-        if not futurized:
+        if sys.version_info.major > 2 and python_template_version < 3 and not futurized:
             # Possibly an error caused by attempting to run python 2
             # template code on python 3. Run the generated module code
             # through futurize and hope for the best.

lib/galaxy/workflow/modules.py

@@ -993,7 +993,9 @@ def get_all_outputs(self, data_only=False):
                         params = make_dict_copy(self.state.inputs)
                         params['on_string'] = 'input dataset(s)'
                         params['tool'] = self.tool
-                        extra_kwds['label'] = fill_template(tool_output.label, context=params)
+                        extra_kwds['label'] = fill_template(tool_output.label,
+                                                            context=params,
+                                                            python_template_version=self.tool.python_template_version)
                     except Exception:
                         pass
                 data_outputs.append(

test/unit/tools/test_evaluation.py

@@ -266,6 +266,7 @@ class MockTool(object):
 
     def __init__(self, app):
         self.profile = 16.01
+        self.python_template_version = 2.7
         self.app = app
         self.hooks_called = []
         self.environment_variables = []

test/unit/tools/test_fill_template.py

@@ -53,7 +53,7 @@ def test_fill_list_comprehension_template():
 
 
 def test_fill_list_comprehension_template_2():
-    template_str = fill_template(LIST_COMPREHENSION_TEMPLATE, retry=1)
+    template_str = fill_template(LIST_COMPREHENSION_TEMPLATE, python_template_version=2, retry=1)
     assert template_str == 'echo 1\n'
 
 
@@ -73,5 +73,5 @@ def test_gen_expr():
 
 
 def test_fix_template_two_to_three():
-    template_str = fill_template(TWO_TO_THREE_TEMPLATE, retry=1)
+    template_str = fill_template(TWO_TO_THREE_TEMPLATE, python_template_version=2, retry=1)
     assert template_str == 'a a 1'