From 4a4306e570671e7c2cb8080c3890848b4f489288 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 3 Oct 2017 14:07:50 -0600 Subject: [PATCH 01/41] ran 2to3 on XML directory --- scripts/lib/CIME/XML/compilerblock.py | 8 ++++---- scripts/lib/CIME/XML/compilers.py | 6 +++--- scripts/lib/CIME/XML/component.py | 4 ++-- scripts/lib/CIME/XML/compsets.py | 6 +++--- scripts/lib/CIME/XML/entry_id.py | 4 ++-- scripts/lib/CIME/XML/env_batch.py | 4 ++-- scripts/lib/CIME/XML/env_mach_specific.py | 6 +++--- scripts/lib/CIME/XML/generic_xml.py | 4 ++-- scripts/lib/CIME/XML/grids.py | 4 ++-- scripts/lib/CIME/XML/machines.py | 10 +++++----- scripts/lib/CIME/XML/test_reporter.py | 14 +++++++------- scripts/lib/CIME/XML/testlist.py | 4 ++-- scripts/lib/CIME/XML/tests.py | 2 +- 13 files changed, 38 insertions(+), 38 deletions(-) diff --git a/scripts/lib/CIME/XML/compilerblock.py b/scripts/lib/CIME/XML/compilerblock.py index c3fee91e4ec..873ed398045 100644 --- a/scripts/lib/CIME/XML/compilerblock.py +++ b/scripts/lib/CIME/XML/compilerblock.py @@ -201,7 +201,7 @@ def _elem_to_setting(self, elem): variables that this setting depends on. """ # Attributes on an element are the conditions on that element. - conditions = dict(elem.items()) + conditions = dict(list(elem.items())) if self._compiler is not None: conditions["COMPILER"] = self._compiler # Deal with internal markup. @@ -225,7 +225,7 @@ def _add_elem_to_lists(self, name, elem, value_lists): of all settings for each variable. """ # Skip this if the element's MPILIB is not valid. - if "MPILIB" in elem.keys() and \ + if "MPILIB" in list(elem.keys()) and \ not self._machobj.is_valid_MPIlib(elem.get("MPILIB")): return setting, depends = self._elem_to_setting(elem) @@ -259,13 +259,13 @@ def matches_machine(self): before add_settings_to_lists if machine-specific output is needed. """ self._specificity = 0 - if "MACH" in self._compiler_elem.keys(): + if "MACH" in list(self._compiler_elem.keys()): if self._machobj.get_machine_name() == \ self._compiler_elem.get("MACH"): self._specificity += 2 else: return False - if "OS" in self._compiler_elem.keys(): + if "OS" in list(self._compiler_elem.keys()): if self._machobj.get_value("OS") == self._compiler_elem.get("OS"): self._specificity += 1 else: diff --git a/scripts/lib/CIME/XML/compilers.py b/scripts/lib/CIME/XML/compilers.py index 41d4bdfeaf3..6ed1c5feef6 100644 --- a/scripts/lib/CIME/XML/compilers.py +++ b/scripts/lib/CIME/XML/compilers.py @@ -160,7 +160,7 @@ def write_macros_file(self, macros_file="Macros.make", output_format="make", xml else: format_ = output_format - if isinstance(macros_file, basestring): + if isinstance(macros_file, str): with open(macros_file, "w") as macros: self._write_macros_file_v2(format_, macros) else: @@ -205,7 +205,7 @@ def _write_macros_file_v2(self, build_system, output, xml=None): while value_lists: # Variables that are ready to be written. ready_variables = [ - var_name for var_name in value_lists.keys() + var_name for var_name in list(value_lists.keys()) if value_lists[var_name].depends <= vars_written ] expect(len(ready_variables) > 0, @@ -254,7 +254,7 @@ def _add_to_macros(node, macros): else: cond_macros = macros["_COND_"] - for key, value2 in attrib.iteritems(): + for key, value2 in attrib.items(): if key not in cond_macros: cond_macros[key] = {} if value2 not in cond_macros[key]: diff --git a/scripts/lib/CIME/XML/component.py b/scripts/lib/CIME/XML/component.py index e0eed6c417b..1f4ae93162c 100644 --- a/scripts/lib/CIME/XML/component.py +++ b/scripts/lib/CIME/XML/component.py @@ -75,7 +75,7 @@ def _get_value_match(self, node, attributes=None, exact_match=False): for valnode in self.get_nodes("value", root=node): # loop through all the keys in valnode (value nodes) attributes - for key,value in valnode.attrib.iteritems(): + for key,value in valnode.attrib.items(): # determine if key is in attributes dictionary match_count = 0 if attributes is not None and key in attributes: @@ -281,6 +281,6 @@ def print_values(self): compsets[attrib] = text logger.info(" {}".format(helptext)) - for v in sorted(compsets.iteritems()): + for v in sorted(compsets.items()): label, definition = v logger.info(" {:20s} : {}".format(label, definition)) diff --git a/scripts/lib/CIME/XML/compsets.py b/scripts/lib/CIME/XML/compsets.py index f883511689b..d120bb30602 100644 --- a/scripts/lib/CIME/XML/compsets.py +++ b/scripts/lib/CIME/XML/compsets.py @@ -84,7 +84,7 @@ def print_values(self, arg_help=True): logger.info(" --------------------------------------") logger.info(" Compset Alias: Compset Long Name ") logger.info(" --------------------------------------") - for key in sorted(compsets_text.iterkeys()): + for key in sorted(compsets_text.keys()): logger.info(" {:20} : {}".format(key, compsets_text[key])) def return_all_values(self): @@ -92,11 +92,11 @@ def return_all_values(self): science_compsets = dict() help_text = self.get_value(name="help") compsets_text = self.get_value("names") - for key in sorted(compsets_text.iterkeys()): + for key in sorted(compsets_text.keys()): all_compsets[key] = compsets_text[key] # get the matching science support grids - for alias in all_compsets.iterkeys(): + for alias in all_compsets.keys(): science_compsets[alias] = self.get_compset_match(alias) return help_text, all_compsets diff --git a/scripts/lib/CIME/XML/entry_id.py b/scripts/lib/CIME/XML/entry_id.py index 94b2aca8423..23cf26f7bb6 100644 --- a/scripts/lib/CIME/XML/entry_id.py +++ b/scripts/lib/CIME/XML/entry_id.py @@ -83,7 +83,7 @@ def _get_value_match(self, node, attributes=None, exact_match=False): # For each node in the list start a score. score = 0 if attributes: - for attribute in vnode.keys(): + for attribute in list(vnode.keys()): # For each attribute, add to the score. score += 1 # If some attribute is specified that we don't know about, @@ -360,7 +360,7 @@ def add_elements_by_group(self, srcobj, attributes=None, infile=None): gname = "group_not_set" # If group with id=$gname does not exist in self.groups # then create the group node and add it to infile file - if gname not in self.groups.keys(): + if gname not in list(self.groups.keys()): newgroup = ET.Element("group") newgroup.set("id",gname) # initialize an empty list diff --git a/scripts/lib/CIME/XML/env_batch.py b/scripts/lib/CIME/XML/env_batch.py index b0ac5843bd7..b0f585f289e 100644 --- a/scripts/lib/CIME/XML/env_batch.py +++ b/scripts/lib/CIME/XML/env_batch.py @@ -123,7 +123,7 @@ def create_job_groups(self, batch_jobs): for name, jdict in batch_jobs: new_job_group = ET.Element("group") new_job_group.set("id", name) - for field in jdict.keys(): + for field in list(jdict.keys()): val = jdict[field] node = ET.SubElement(new_job_group, "entry", {"id":field,"value":val}) tnode = ET.SubElement(node, "type") @@ -359,7 +359,7 @@ def submit_jobs(self, case, no_batch=False, job=None, skip_pnl=False, if self._prereq_jobid is not None: jobid = self._prereq_jobid for dep in deps: - if dep in depid.keys() and depid[dep] is not None: + if dep in list(depid.keys()) and depid[dep] is not None: jobid += " " + str(depid[dep]) #TODO: doubt these will be used # elif dep == "and": diff --git a/scripts/lib/CIME/XML/env_mach_specific.py b/scripts/lib/CIME/XML/env_mach_specific.py index 64fa53c66de..0bea8710df0 100644 --- a/scripts/lib/CIME/XML/env_mach_specific.py +++ b/scripts/lib/CIME/XML/env_mach_specific.py @@ -258,9 +258,9 @@ def _load_modules_generic(self, modules_to_load): cmd = "source {}".format(sh_init_cmd) - if os.environ.has_key("SOFTENV_ALIASES"): + if "SOFTENV_ALIASES" in os.environ: cmd += " && source $SOFTENV_ALIASES" - if os.environ.has_key("SOFTENV_LOAD"): + if "SOFTENV_LOAD" in os.environ: cmd += " && source $SOFTENV_LOAD" for action,argument in modules_to_load: @@ -356,7 +356,7 @@ def get_mpirun(self, case, attribs, job="case.run", exe_only=False): matches = 0 is_default = False - for key, value in attribs.iteritems(): + for key, value in attribs.items(): expect(key in self._allowed_mpi_attributes, "Unexpected key {} in mpirun attributes".format(key)) if key in xml_attribs: if xml_attribs[key].lower() == "false": diff --git a/scripts/lib/CIME/XML/generic_xml.py b/scripts/lib/CIME/XML/generic_xml.py index efad45fa2ee..6ad497770a6 100644 --- a/scripts/lib/CIME/XML/generic_xml.py +++ b/scripts/lib/CIME/XML/generic_xml.py @@ -123,9 +123,9 @@ def get_nodes(self, nodename, attributes=None, root=None, xpath=None): # one attribute in an xpath query so we query seperately for each attribute # and create a result with the intersection of those lists - for key, value in attributes.iteritems(): + for key, value in attributes.items(): if value is not None: - expect(isinstance(value, basestring), + expect(isinstance(value, str), " Bad value passed for key {}".format(key)) xpath = ".//{}[@{}=\'{}\']".format(nodename, key, value) logger.debug("xpath is {}".format(xpath)) diff --git a/scripts/lib/CIME/XML/grids.py b/scripts/lib/CIME/XML/grids.py index 634a4be28eb..b443ef06c37 100644 --- a/scripts/lib/CIME/XML/grids.py +++ b/scripts/lib/CIME/XML/grids.py @@ -533,7 +533,7 @@ def _print_values_v1(self, long_output=None): logger.info(" short name: {}".format(sname)) if alias is not None: logger.info(" alias: {}".format(alias)) - for attr, value in grid_node.items(): + for attr, value in list(grid_node.items()): if attr == 'compset': logger.info(" compset match: {}".format(value)) @@ -582,7 +582,7 @@ def _get_all_values_v1(self): "alias" : alias, } compset_list = list() - for attr, value in grid_node.items(): + for attr, value in list(grid_node.items()): if attr == 'compset': compset_list.append(value) diff --git a/scripts/lib/CIME/XML/machines.py b/scripts/lib/CIME/XML/machines.py index fc39ac556b4..d762b37ff89 100644 --- a/scripts/lib/CIME/XML/machines.py +++ b/scripts/lib/CIME/XML/machines.py @@ -300,13 +300,13 @@ def print_values(self): max_tasks_per_node = machine.find("MAX_TASKS_PER_NODE") MAX_MPITASKS_PER_NODE = machine.find("MAX_MPITASKS_PER_NODE") - print( " {} : {} ".format(name , desc.text)) - print( " os ", os_.text) - print( " compilers ",compilers.text) + print(( " {} : {} ".format(name , desc.text))) + print(( " os ", os_.text)) + print(( " compilers ",compilers.text)) if MAX_MPITASKS_PER_NODE is not None: - print( " pes/node ",MAX_MPITASKS_PER_NODE.text) + print(( " pes/node ",MAX_MPITASKS_PER_NODE.text)) if max_tasks_per_node is not None: - print( " max_tasks/node ",max_tasks_per_node.text) + print(( " max_tasks/node ",max_tasks_per_node.text)) def return_all_values(self): # return a dictionary of machines diff --git a/scripts/lib/CIME/XML/test_reporter.py b/scripts/lib/CIME/XML/test_reporter.py index 1dd841bc6a2..ca63481c2ae 100644 --- a/scripts/lib/CIME/XML/test_reporter.py +++ b/scripts/lib/CIME/XML/test_reporter.py @@ -7,7 +7,7 @@ from CIME.XML.generic_xml import GenericXML from CIME.utils import expect,get_model -import urllib +import urllib.request, urllib.parse, urllib.error @@ -97,17 +97,17 @@ def push2testdb(self): # Post test result XML to CESM test database # xmlstr = ET.tostring(self.root,method="xml",encoding="UTF-8") - username=raw_input("Username:") + username=input("Username:") os.system("stty -echo") - password=raw_input("Password:") + password=input("Password:") os.system("stty echo") params={'username':username,'password':password,'testXML':xmlstr} url="https://csegweb.cgd.ucar.edu/testdb/cgi-bin/processXMLtest.cgi" - params = urllib.urlencode(params) - f = urllib.urlopen(url, params) + params = urllib.parse.urlencode(params) + f = urllib.request.urlopen(url, params) # # Print any messages from the post command # - print(f.read()) - print(f.code) + print((f.read())) + print((f.code)) diff --git a/scripts/lib/CIME/XML/testlist.py b/scripts/lib/CIME/XML/testlist.py index c1fd4c8d48f..bc3e4f7a49b 100644 --- a/scripts/lib/CIME/XML/testlist.py +++ b/scripts/lib/CIME/XML/testlist.py @@ -115,7 +115,7 @@ def _get_testsv2(self, machine=None, category=None, compiler=None, compset=None, machnodes = self.get_nodes("machine",machatts,root=tnode) if machnodes: this_test_node = {} - for key, value in tnode.items(): + for key, value in list(tnode.items()): if key == "name": this_test_node["testname"] = value else: @@ -124,7 +124,7 @@ def _get_testsv2(self, machine=None, category=None, compiler=None, compset=None, for mach in machnodes: # this_test_node can include multiple tests this_test = dict(this_test_node) - for key, value in mach.items(): + for key, value in list(mach.items()): if key == "name": this_test["machine"] = value else: diff --git a/scripts/lib/CIME/XML/tests.py b/scripts/lib/CIME/XML/tests.py index 944f6934687..5c5351543d2 100644 --- a/scripts/lib/CIME/XML/tests.py +++ b/scripts/lib/CIME/XML/tests.py @@ -50,4 +50,4 @@ def print_values(self, skip_infrastructure_tests=True): continue name = one_test.get("NAME") desc = one_test.find("DESC").text - print("{}: {}".format(name, desc)) + print(("{}: {}".format(name, desc))) From 4db671c8816ab53856d00ccae511b9a8d6b2081b Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 3 Oct 2017 14:09:12 -0600 Subject: [PATCH 02/41] ran 2to3 on SystemTests dir --- scripts/lib/CIME/SystemTests/nodefail.py | 2 +- scripts/lib/CIME/SystemTests/system_tests_common.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/lib/CIME/SystemTests/nodefail.py b/scripts/lib/CIME/SystemTests/nodefail.py index 4a1204f8876..daf5a0f8d5a 100644 --- a/scripts/lib/CIME/SystemTests/nodefail.py +++ b/scripts/lib/CIME/SystemTests/nodefail.py @@ -49,7 +49,7 @@ def _restart_fake_phase(self): with open(fake_exe_file, "w") as fd: fd.write(fake_exe) - os.chmod(fake_exe_file, 0755) + os.chmod(fake_exe_file, 0o755) prev_run_exe = self._case.get_value("run_exe") env_mach_specific = self._case.get_env("mach_specific") diff --git a/scripts/lib/CIME/SystemTests/system_tests_common.py b/scripts/lib/CIME/SystemTests/system_tests_common.py index 6f315d238bc..63db522869f 100644 --- a/scripts/lib/CIME/SystemTests/system_tests_common.py +++ b/scripts/lib/CIME/SystemTests/system_tests_common.py @@ -343,12 +343,12 @@ def compare_env_run(self, expected=None): f1obj = EnvRun(self._caseroot, "env_run.xml", components=components) f2obj = EnvRun(self._caseroot, os.path.join(LOCKED_DIR, "env_run.orig.xml"), components=components) diffs = f1obj.compare_xml(f2obj) - for key in diffs.keys(): + for key in list(diffs.keys()): if expected is not None and key in expected: logging.warn(" Resetting {} for test".format(key)) f1obj.set_value(key, f2obj.get_value(key, resolved=False)) else: - print("WARNING: Found difference in test {}: case: {} original value {}".format(key, diffs[key][0], diffs[key][1])) + print(("WARNING: Found difference in test {}: case: {} original value {}".format(key, diffs[key][0], diffs[key][1]))) return False return True @@ -463,7 +463,7 @@ def build_phase(self, sharedlib_only=False, model_only=False): f.write("#!/bin/bash\n") f.write(self._script) - os.chmod(modelexe, 0755) + os.chmod(modelexe, 0o755) build.post_build(self._case, []) From 848d7485b176519d0b9ce9d38d34d3704a331630 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 3 Oct 2017 14:11:02 -0600 Subject: [PATCH 03/41] ran 2to3 on BuildTools directory --- .../lib/CIME/BuildTools/macroconditiontree.py | 4 ++-- scripts/lib/CIME/BuildTools/macrowriterbase.py | 16 +++++++--------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/scripts/lib/CIME/BuildTools/macroconditiontree.py b/scripts/lib/CIME/BuildTools/macroconditiontree.py index 4172f2621f5..e04721d48b3 100644 --- a/scripts/lib/CIME/BuildTools/macroconditiontree.py +++ b/scripts/lib/CIME/BuildTools/macroconditiontree.py @@ -32,7 +32,7 @@ def __init__(self, name, settings): # To make merging more effective, sort the conditions. all_conditions = [] for setting in settings: - all_conditions += setting.conditions.keys() + all_conditions += list(setting.conditions.keys()) if all_conditions: condition = sorted(all_conditions)[0] if condition is None: @@ -119,7 +119,7 @@ def merge(self, other): return other.merge(self) # If neither is a leaf and their conditions match, merge # their sets of branches. - for (cond_val, other_branch) in other._branches.items(): + for (cond_val, other_branch) in list(other._branches.items()): if cond_val in self._branches: self._branches[cond_val] = \ self._branches[cond_val].merge(other_branch) diff --git a/scripts/lib/CIME/BuildTools/macrowriterbase.py b/scripts/lib/CIME/BuildTools/macrowriterbase.py index db00e8383c4..c0d33fc9071 100644 --- a/scripts/lib/CIME/BuildTools/macrowriterbase.py +++ b/scripts/lib/CIME/BuildTools/macrowriterbase.py @@ -69,7 +69,7 @@ def _get_components(value): return components -class MacroWriterBase(object): +class MacroWriterBase(object, metaclass=ABCMeta): """Abstract base class for macro file writers. @@ -101,8 +101,6 @@ class MacroWriterBase(object): end_ifeq """ - __metaclass__ = ABCMeta - indent_increment = 2 def __init__(self, output): @@ -132,7 +130,7 @@ def write_line(self, line): A trailing newline is added, whether or not the input has one. """ - self.output.write(unicode(self.indent_string() + line + "\n")) + self.output.write(str(self.indent_string() + line + "\n")) @abstractmethod def environment_variable_string(self, name): @@ -222,7 +220,7 @@ def write_macros_file_v1(macros, compiler, os_, machine, macros_file="Macros", o fd.write("#\n# Makefile Macros \n") # print the settings out to the Macros file - for key, value in sorted(macros.iteritems()): + for key, value in sorted(macros.items()): if key == "_COND_": pass elif key.startswith("ADD_"): @@ -248,7 +246,7 @@ def write_macros_file_v1(macros, compiler, os_, machine, macros_file="Macros", o # print the settings out to the Macros file, do it in # two passes so that path values appear first in the # file. - for key, value in sorted(macros.iteritems()): + for key, value in sorted(macros.items()): if key == "_COND_": pass else: @@ -262,7 +260,7 @@ def write_macros_file_v1(macros, compiler, os_, machine, macros_file="Macros", o fd.write("set({} {})\n".format(cmake_var, value)) fd.write("list(APPEND CMAKE_PREFIX_PATH {})\n\n".format(value)) - for key, value in sorted(macros.iteritems()): + for key, value in sorted(macros.items()): if key == "_COND_": pass else: @@ -301,10 +299,10 @@ def write_macros_file_v1(macros, compiler, os_, machine, macros_file="Macros", o def _parse_hash(macros, fd, depth, output_format, cmakedebug=""): width = 2 * depth - for key, value in macros.iteritems(): + for key, value in macros.items(): if type(value) is dict: if output_format == "make" or "DEBUG" in key: - for key2, value2 in value.iteritems(): + for key2, value2 in value.items(): if output_format == "make": fd.write("{}ifeq ($({}), {}) \n".format(" " * width, key, key2)) From 8e74e19a14f4ad38cbf226edf9806cac39e3f11f Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 3 Oct 2017 14:11:32 -0600 Subject: [PATCH 04/41] ran 2to3 on CIME directory --- scripts/lib/CIME/aprun.py | 4 ++-- scripts/lib/CIME/bless_test_results.py | 10 ++++----- scripts/lib/CIME/build.py | 2 +- scripts/lib/CIME/buildlib.py | 2 +- scripts/lib/CIME/case.py | 20 +++++++++--------- scripts/lib/CIME/case_setup.py | 2 +- scripts/lib/CIME/case_submit.py | 2 +- scripts/lib/CIME/check_lockedfiles.py | 8 ++++---- scripts/lib/CIME/compare_namelists.py | 8 ++++---- scripts/lib/CIME/compare_test_results.py | 2 +- scripts/lib/CIME/get_timing.py | 14 ++++++------- scripts/lib/CIME/hist_utils.py | 2 +- scripts/lib/CIME/namelist.py | 26 ++++++++++++------------ scripts/lib/CIME/nmlgen.py | 2 +- scripts/lib/CIME/test_scheduler.py | 8 ++++---- scripts/lib/CIME/test_status.py | 8 ++++---- scripts/lib/CIME/user_mod_support.py | 2 +- scripts/lib/CIME/utils.py | 6 +++--- scripts/lib/CIME/wait_for_tests.py | 8 ++++---- 19 files changed, 68 insertions(+), 68 deletions(-) diff --git a/scripts/lib/CIME/aprun.py b/scripts/lib/CIME/aprun.py index c09574912d2..8ffdd916201 100755 --- a/scripts/lib/CIME/aprun.py +++ b/scripts/lib/CIME/aprun.py @@ -65,14 +65,14 @@ def _get_aprun_cmd_for_case_impl(ntasks, nthreads, rootpes, pstrids, c2 += 1 # make sure all maxt values at least 1 - for c1 in xrange(0, total_tasks): + for c1 in range(0, total_tasks): if maxt[c1] < 1: maxt[c1] = 1 # Compute task and thread settings for batch commands tasks_per_node, task_count, thread_count, max_thread_count, total_node_count, aprun_args = \ 0, 1, maxt[0], maxt[0], 0, "" - for c1 in xrange(1, total_tasks): + for c1 in range(1, total_tasks): if maxt[c1] != thread_count: tasks_per_node = min(MAX_MPITASKS_PER_NODE, max_tasks_per_node / thread_count) diff --git a/scripts/lib/CIME/bless_test_results.py b/scripts/lib/CIME/bless_test_results.py index b89b21a6826..d6a1ace19d9 100644 --- a/scripts/lib/CIME/bless_test_results.py +++ b/scripts/lib/CIME/bless_test_results.py @@ -15,9 +15,9 @@ def bless_namelists(test_name, report_only, force, baseline_name, baseline_root) # re-run create_test. # Update namelist files - print("Test '{}' had namelist diff".format(test_name)) + print(("Test '{}' had namelist diff".format(test_name))) if (not report_only and - (force or raw_input("Update namelists (y/n)? ").upper() in ["Y", "YES"])): + (force or input("Update namelists (y/n)? ").upper() in ["Y", "YES"])): create_test_gen_args = " -g {} ".format(baseline_name if get_model() == "cesm" else " -g -b {} ".format(baseline_name)) stat, _, err = run_cmd("{}/create_test {} -n {} --baseline-root {} -o".format(get_scripts_root(), test_name, create_test_gen_args, baseline_root)) if stat != 0: @@ -42,7 +42,7 @@ def bless_history(test_name, testcase_dir_for_test, baseline_name, baseline_root else: print(comments) if (not report_only and - (force or raw_input("Update this diff (y/n)? ").upper() in ["Y", "YES"])): + (force or input("Update this diff (y/n)? ").upper() in ["Y", "YES"])): result, comments = generate_baseline(case, baseline_dir=baseline_full_dir) if not result: logging.warning("Hist file bless FAILED for test {}".format(test_name)) @@ -97,11 +97,11 @@ def bless_test_results(baseline_name, baseline_root, test_root, compiler, test_i # Now, do the bless if not nl_bless and not hist_bless: - print("Nothing to bless for test:", test_name, " overall status:", overall_result) + print(("Nothing to bless for test:", test_name, " overall status:", overall_result)) else: print("###############################################################################") - print("Blessing results for test:", test_name, "most recent result:", overall_result) + print(("Blessing results for test:", test_name, "most recent result:", overall_result)) print("###############################################################################") if not force: time.sleep(2) diff --git a/scripts/lib/CIME/build.py b/scripts/lib/CIME/build.py index bf7b3909f51..b46b9effa8c 100644 --- a/scripts/lib/CIME/build.py +++ b/scripts/lib/CIME/build.py @@ -387,7 +387,7 @@ def _case_build_impl(caseroot, case, sharedlib_only, model_only, buildlist): # Retrieve relevant case data # This environment variable gets set for cesm Make and # needs to be unset before building again. - if "MODEL" in os.environ.keys(): + if "MODEL" in list(os.environ.keys()): del os.environ["MODEL"] build_threaded = case.get_build_threaded() casetools = case.get_value("CASETOOLS") diff --git a/scripts/lib/CIME/buildlib.py b/scripts/lib/CIME/buildlib.py index d04860bce39..35d5baf94fc 100644 --- a/scripts/lib/CIME/buildlib.py +++ b/scripts/lib/CIME/buildlib.py @@ -89,4 +89,4 @@ def run_gmake(case, compclass, libroot, bldroot, libname="", user_cppdefs=""): rc, out, err = run_cmd(cmd) expect(rc == 0, "Command {} failed rc={:d}\nout={}\nerr={}".format(cmd, rc, out, err)) - print "Command {} completed with output {}\nerr {}".format(cmd, out, err) + print("Command {} completed with output {}\nerr {}".format(cmd, out, err)) diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index 997c3a4c2d9..67fdd40cdd7 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -402,7 +402,7 @@ def set_valid_values(self, item, valid_values): return result def set_lookup_value(self, item, value): - if item in self.lookups.keys() and self.lookups[item] is not None: + if item in list(self.lookups.keys()) and self.lookups[item] is not None: logger.warn("Item {} already in lookups with value {}".format(item,self.lookups[item])) else: logger.debug("Setting in lookups: item {}, value {}".format(item,value)) @@ -410,7 +410,7 @@ def set_lookup_value(self, item, value): def clean_up_lookups(self, allow_undefined=False): # put anything in the lookups table into existing env objects - for key,value in self.lookups.items(): + for key,value in list(self.lookups.items()): logger.debug("lookup key {} value {}".format(key, value)) result = self.set_value(key,value, allow_undefined=allow_undefined) if result is not None: @@ -614,7 +614,7 @@ def _get_component_config_data(self, files): if len(self._component_classes) > len(self._components): self._components.append('sesp') - for i in xrange(1,len(self._component_classes)): + for i in range(1,len(self._component_classes)): comp_class = self._component_classes[i] comp_name = self._components[i-1] node_name = 'CONFIG_' + comp_class + '_FILE' @@ -694,7 +694,7 @@ def _setup_mach_pes(self, pecount, multi_driver, ninst, machine_name, mpilib): mach_pes_obj = self.get_env("mach_pes") if other is not None: - for key, value in other.items(): + for key, value in list(other.items()): self.set_value(key, value) totaltasks = [] @@ -733,10 +733,10 @@ def _setup_mach_pes(self, pecount, multi_driver, ninst, machine_name, mpilib): mach_pes_obj.set_value(key, ninst) key = "NTASKS_{}".format(compclass) - if key not in pes_ntasks.keys(): + if key not in list(pes_ntasks.keys()): mach_pes_obj.set_value(key,1) key = "NTHRDS_{}".format(compclass) - if compclass not in pes_nthrds.keys(): + if compclass not in list(pes_nthrds.keys()): mach_pes_obj.set_value(compclass,1) return pesize @@ -768,7 +768,7 @@ def configure(self, compset_name, grid_name, machine_name=None, gridinfo = grids.get_grid_info(name=grid_name, compset=self._compsetname) self._gridname = gridinfo["GRID"] - for key,value in gridinfo.items(): + for key,value in list(gridinfo.items()): logger.debug("Set grid {} {}".format(key,value)) self.set_lookup_value(key,value) @@ -887,7 +887,7 @@ def configure(self, compset_name, grid_name, machine_name=None, if os.path.exists(wdir): expect(not test, "Directory {} already exists, aborting test".format(wdir)) if answer is None: - response = raw_input("\nDirectory {} already exists, (r)eplace, (a)bort, or (u)se existing?".format(wdir)) + response = input("\nDirectory {} already exists, (r)eplace, (a)bort, or (u)se existing?".format(wdir)) else: response = answer @@ -967,7 +967,7 @@ def _set_pio_xml(self): mpilib = self.get_value("MPILIB") defaults = pioobj.get_defaults(grid=grid,compset=compset,mach=mach,compiler=compiler, mpilib=mpilib) - for vid, value in defaults.items(): + for vid, value in list(defaults.items()): self.set_value(vid,value) def _create_caseroot_tools(self): @@ -1176,7 +1176,7 @@ def report_job_status(self): if not jobmap: logger.info("No job ids associated with this case. Either case.submit was not run or was run with no-batch") else: - for jobname, jobid in jobmap.iteritems(): + for jobname, jobid in jobmap.items(): status = self.get_env("batch").get_status(jobid) if status: logger.info("{}: {}".format(jobname, status)) diff --git a/scripts/lib/CIME/case_setup.py b/scripts/lib/CIME/case_setup.py index 4c2750412e9..ad71e8482ca 100644 --- a/scripts/lib/CIME/case_setup.py +++ b/scripts/lib/CIME/case_setup.py @@ -49,7 +49,7 @@ def _build_usernl_files(case, model, comp): nlfile = "user_nl_{}".format(comp) model_nl = os.path.join(model_dir, nlfile) if ninst > 1: - for inst_counter in xrange(1, ninst+1): + for inst_counter in range(1, ninst+1): inst_nlfile = "{}_{:04d}".format(nlfile, inst_counter) if not os.path.exists(inst_nlfile): # If there is a user_nl_foo in the case directory, copy it diff --git a/scripts/lib/CIME/case_submit.py b/scripts/lib/CIME/case_submit.py index 90176e85571..80141ebeb6f 100644 --- a/scripts/lib/CIME/case_submit.py +++ b/scripts/lib/CIME/case_submit.py @@ -68,7 +68,7 @@ def _submit(case, job=None, resubmit=False, no_batch=False, skip_pnl=False, batch_args=batch_args) xml_jobids = [] - for jobname, jobid in job_ids.iteritems(): + for jobname, jobid in job_ids.items(): logger.info("Submitted job {} with id {}".format(jobname, jobid)) if jobid: xml_jobids.append("{}:{}".format(jobname, jobid)) diff --git a/scripts/lib/CIME/check_lockedfiles.py b/scripts/lib/CIME/check_lockedfiles.py index f2bc049385d..b4c582a2493 100644 --- a/scripts/lib/CIME/check_lockedfiles.py +++ b/scripts/lib/CIME/check_lockedfiles.py @@ -112,9 +112,9 @@ def check_lockedfiles(case): diffs = f1obj.compare_xml(f2obj) if diffs: logging.warn("File {} has been modified".format(lfile)) - for key in diffs.keys(): - print(" found difference in {} : case {} locked {}" - .format(key, repr(diffs[key][0]), repr(diffs[key][1]))) + for key in list(diffs.keys()): + print((" found difference in {} : case {} locked {}" + .format(key, repr(diffs[key][0]), repr(diffs[key][1])))) if objname == "env_mach_pes": expect(False, "Invoke case.setup --reset ") @@ -124,7 +124,7 @@ def check_lockedfiles(case): elif objname == "env_build": logging.warn("Setting build complete to False") case.set_value("BUILD_COMPLETE", False) - if "PIO_VERSION" in diffs.keys(): + if "PIO_VERSION" in list(diffs.keys()): case.set_value("BUILD_STATUS", 2) logging.critical("Changing PIO_VERSION requires running " "case.build --clean-all and rebuilding") diff --git a/scripts/lib/CIME/compare_namelists.py b/scripts/lib/CIME/compare_namelists.py index b8d75648027..bf8f2f3af3c 100644 --- a/scripts/lib/CIME/compare_namelists.py +++ b/scripts/lib/CIME/compare_namelists.py @@ -353,7 +353,7 @@ def _compare_values(name, gold_value, comp_value, case): comments += " list variable '{}' has extra value {}\n".format(name, comp_value_list_item) elif (type(gold_value) is OrderedDict): - for key, gold_value_dict_item in gold_value.iteritems(): + for key, gold_value_dict_item in gold_value.items(): if (key in comp_value): comments += _compare_values("{} dict item {}".format(name, key), gold_value_dict_item, comp_value[key], case) @@ -493,12 +493,12 @@ def _compare_namelists(gold_namelists, comp_namelists, case): '' """ different_namelists = OrderedDict() - for namelist, gold_names in gold_namelists.iteritems(): + for namelist, gold_names in gold_namelists.items(): if (namelist not in comp_namelists): different_namelists[namelist] = ["Missing namelist: {}\n".format(namelist)] else: comp_names = comp_namelists[namelist] - for name, gold_value in gold_names.iteritems(): + for name, gold_value in gold_names.items(): if (name not in comp_names): different_namelists.setdefault(namelist, []).append(" missing variable: '{}'\n".format(name)) else: @@ -516,7 +516,7 @@ def _compare_namelists(gold_namelists, comp_namelists, case): different_namelists[namelist] = ["Found extra namelist: {}\n".format(namelist)] comments = "" - for namelist, nlcomment in different_namelists.iteritems(): + for namelist, nlcomment in different_namelists.items(): if len(nlcomment) == 1: comments += nlcomment[0] else: diff --git a/scripts/lib/CIME/compare_test_results.py b/scripts/lib/CIME/compare_test_results.py index ef476df35e0..4964cff96a4 100644 --- a/scripts/lib/CIME/compare_test_results.py +++ b/scripts/lib/CIME/compare_test_results.py @@ -152,7 +152,7 @@ def compare_test_results(baseline_name, baseline_root, test_root, compiler, test brief_result += " {}".format(compare_comment) brief_result += "\n" - print(brief_result,) + print((brief_result,)) append_status_cprnc_log(brief_result, logfile_name, test_dir) diff --git a/scripts/lib/CIME/get_timing.py b/scripts/lib/CIME/get_timing.py index 2e128bf56ad..38b6153b353 100644 --- a/scripts/lib/CIME/get_timing.py +++ b/scripts/lib/CIME/get_timing.py @@ -145,7 +145,7 @@ def _getTiming(self, inst=0): else: pecost = totalpes - for m in self.models.values(): + for m in list(self.models.values()): for key in ["NTASKS", "ROOTPE", "PSTRID", "NTHRDS", "NINST"]: if key == "NINST" and m.name == "CPL": m.ninst = 1 @@ -179,7 +179,7 @@ def _getTiming(self, inst=0): try: shutil.copyfile(binfilename, finfilename) - except Exception, e: + except Exception as e: if not os.path.isfile(binfilename): logger.critical("File {} not found".format(binfilename)) else: @@ -191,7 +191,7 @@ def _getTiming(self, inst=0): fin = open(finfilename, "r") self.finlines = fin.readlines() fin.close() - except Exception, e: + except Exception as e: logger.critical("Unable to open file {}".format(finfilename)) raise e @@ -215,7 +215,7 @@ def _getTiming(self, inst=0): if inittype == "TRUE": odays = odays - (tlen/ocn_ncpl) - peminmax = max([m.rootpe for m in self.models.values()])+1 + peminmax = max([m.rootpe for m in list(self.models.values())])+1 if ncpl_base_period in ["decade","year","day"] and int(adays) > 0: adays = int(adays) if tlen % ocn_ncpl == 0: @@ -223,12 +223,12 @@ def _getTiming(self, inst=0): self.adays = adays maxoffset = 40 extraoff = 20 - for m in self.models.values(): + for m in list(self.models.values()): m.offset = int((maxoffset*m.rootpe)/peminmax) + extraoff cpl.offset = 0 try: self.fout = open(foutfilename, "w") - except Exception, e: + except Exception as e: logger.critical("Could not open file for writing: {}".format(foutfilename)) raise e @@ -293,7 +293,7 @@ def _getTiming(self, inst=0): tmax = tmax + wtmin + correction ocn.tmax += ocnrunitime - for m in self.models.values(): + for m in list(self.models.values()): m.tmaxr = 0 if m.tmax > 0: m.tmaxr = adays*86400.0/(m.tmax*365.0) diff --git a/scripts/lib/CIME/hist_utils.py b/scripts/lib/CIME/hist_utils.py index bb9714683ee..59cdd837a87 100644 --- a/scripts/lib/CIME/hist_utils.py +++ b/scripts/lib/CIME/hist_utils.py @@ -44,7 +44,7 @@ def _get_latest_hist_files(testcase, model, from_dir, suffix=""): ext = get_extension(model, hist) latest_files[ext] = hist - for key in latest_files.keys(): + for key in list(latest_files.keys()): histlist.append(latest_files[key]) return histlist diff --git a/scripts/lib/CIME/namelist.py b/scripts/lib/CIME/namelist.py index f3115a1845e..5e31a95d35b 100644 --- a/scripts/lib/CIME/namelist.py +++ b/scripts/lib/CIME/namelist.py @@ -732,21 +732,21 @@ def compress_literal_list(literals): # Otherwise, write out the previous literal and start tracking the # new one. rep_str = str(num_reps) + '*' if num_reps > 1 else '' - if isinstance(old_literal, basestring): + if isinstance(old_literal, str): compressed.append(rep_str + old_literal) else: compressed.append(rep_str + str(old_literal)) old_literal = literal num_reps = 1 rep_str = str(num_reps) + '*' if num_reps > 1 else '' - if isinstance(old_literal, basestring): + if isinstance(old_literal, str): compressed.append(rep_str + old_literal) else: compressed.append(rep_str + str(old_literal)) return compressed else: for literal in literals: - if isinstance(literal, basestring): + if isinstance(literal, str): compressed.append(literal) else: compressed.append(str(literal)) @@ -833,7 +833,7 @@ def parse(in_file=None, text=None, groupless=False, convert_tab_to_space=True): "Must specify an input file or text to the namelist parser.") expect(in_file is None or text is None, "Cannot specify both input file and text to the namelist parser.") - if isinstance(in_file, str) or isinstance(in_file, unicode): + if isinstance(in_file, str) or isinstance(in_file, str): logger.debug("Reading namelist at: {}".format(in_file)) with open(in_file) as in_file_obj: text = in_file_obj.read() @@ -901,7 +901,7 @@ def get_group_names(self): >>> sorted(parse(text='&foo / &bar /').get_group_names()) [u'bar', u'foo'] """ - return self._groups.keys() + return list(self._groups.keys()) def get_variable_names(self, group_name): """Return a list of all variables in the given namelist group. @@ -923,7 +923,7 @@ def get_variable_names(self, group_name): group_name = group_name.lower() if group_name not in self._groups: return [] - return self._groups[group_name].keys() + return list(self._groups[group_name].keys()) def get_variable_value(self, group_name, variable_name): """Return the value of the specified variable. @@ -943,7 +943,7 @@ def get_variable_value(self, group_name, variable_name): variable_name = variable_name.lower() if group_name not in self._groups or \ variable_name not in self._groups[group_name]: - return [u''] + return [''] return self._groups[group_name][variable_name] @@ -974,7 +974,7 @@ def get_value(self, variable_name): if possible_groups: return self._groups[possible_groups[0]][variable_name] else: - return [u''] + return [''] def set_variable_value(self, group_name, variable_name, value, var_size=1): """Set the value of the specified variable. @@ -1134,7 +1134,7 @@ def write(self, out_file, groups=None, append=False, format_='nml', sorted_group """ expect(format_ in ('nml', 'rc', 'nmlcontents'), "Namelist.write: unexpected output format {!r}".format(str(format_))) - if isinstance(out_file, str) or isinstance(out_file, unicode): + if isinstance(out_file, str) or isinstance(out_file, str): logger.debug("Writing namelist to: {}".format(out_file)) flag = 'a' if append else 'w' with open(out_file, flag) as file_obj: @@ -1146,7 +1146,7 @@ def write(self, out_file, groups=None, append=False, format_='nml', sorted_group def _write(self, out_file, groups, format_, sorted_groups): """Unwrapped version of `write` assuming that a file object is input.""" if groups is None: - groups = self._groups.keys() + groups = list(self._groups.keys()) if format_ == 'nml' or format_ == 'nmlcontents': equals = ' =' elif format_ == 'rc': @@ -1249,7 +1249,7 @@ def __init__(self, text, groupless=False): self._line = 1 self._col = 0 # Text and its size. - self._text = unicode(text) + self._text = str(text) self._len = len(self._text) # Dictionary with group names as keys, and dictionaries of variable # name-value pairs as values. (Or a single flat dictionary if @@ -1800,7 +1800,7 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): """ # Deal with empty input string. if allow_eof_end and self._pos == self._len: - return u'' + return '' # Deal with a repeated value prefix. old_pos = self._pos if FORTRAN_REPEAT_PREFIX_REGEX.search(self._text[self._pos:]): @@ -2029,7 +2029,7 @@ def _parse_name_and_values(self, allow_eof_end=False): except _NamelistEOF: # If we hit the end of file, return a name assigned to a null value. if allow_eof_end: - return name, [u''], addto + return name, [''], addto else: raise # Expect at least one literal, even if it's a null value. diff --git a/scripts/lib/CIME/nmlgen.py b/scripts/lib/CIME/nmlgen.py index 0f698f1255b..4a3fd1c6e06 100644 --- a/scripts/lib/CIME/nmlgen.py +++ b/scripts/lib/CIME/nmlgen.py @@ -329,7 +329,7 @@ def _sub_fields(self, varnames): if self._case.get_value('GLC_NEC') == 0: glc_nec_indices = [0] else: - glc_nec_indices = range(self._case.get_value('GLC_NEC')) + glc_nec_indices = list(range(self._case.get_value('GLC_NEC'))) glc_nec_indices.append(glc_nec_indices[-1] + 1) glc_nec_indices.pop(0) for i in glc_nec_indices: diff --git a/scripts/lib/CIME/test_scheduler.py b/scripts/lib/CIME/test_scheduler.py index 6a5e0c8f04c..16180ed8384 100644 --- a/scripts/lib/CIME/test_scheduler.py +++ b/scripts/lib/CIME/test_scheduler.py @@ -249,7 +249,7 @@ def __init__(self, test_names, test_data=None, ########################################################################### def get_testnames(self): - return self._tests.keys() + return list(self._tests.keys()) ########################################################################### def _log_output(self, test, output): @@ -643,7 +643,7 @@ def _get_procs_needed(self, test, phase, threads_in_flight=None, no_batch=False) # Will force serialization of sharedlib builds # TODO - instead of serializing, compute all library configs needed and build # them all in parallel - for _, _, running_phase in threads_in_flight.values(): + for _, _, running_phase in list(threads_in_flight.values()): if (running_phase == SHAREDLIB_BUILD_PHASE): return self._proc_pool + 1 @@ -660,7 +660,7 @@ def _wait_for_something_to_finish(self, threads_in_flight): expect(len(threads_in_flight) <= self._parallel_jobs, "Oversubscribed?") finished_tests = [] while not finished_tests: - for test, thread_info in threads_in_flight.iteritems(): + for test, thread_info in threads_in_flight.items(): if not thread_info[0].is_alive(): finished_tests.append((test, thread_info[1])) @@ -774,7 +774,7 @@ def _producer(self): # No free resources, wait for something in flight to finish self._wait_for_something_to_finish(threads_in_flight) - for unfinished_thread, _, _ in threads_in_flight.values(): + for unfinished_thread, _, _ in list(threads_in_flight.values()): unfinished_thread.join() ########################################################################### diff --git a/scripts/lib/CIME/test_status.py b/scripts/lib/CIME/test_status.py index d560804dc11..87cb239bc1a 100644 --- a/scripts/lib/CIME/test_status.py +++ b/scripts/lib/CIME/test_status.py @@ -132,7 +132,7 @@ def __exit__(self, *_): self.flush() def __iter__(self): - for phase, data in self._phase_statuses.iteritems(): + for phase, data in self._phase_statuses.items(): yield phase, data[0] def get_name(self): @@ -192,7 +192,7 @@ def set_status(self, phase, status, comments=""): if subsequent_phase in self._phase_statuses: del self._phase_statuses[subsequent_phase] if subsequent_phase.startswith(COMPARE_PHASE): - for stored_phase in self._phase_statuses.keys(): + for stored_phase in list(self._phase_statuses.keys()): if stored_phase.startswith(COMPARE_PHASE): del self._phase_statuses[stored_phase] @@ -215,7 +215,7 @@ def phase_statuses_dump(self, prefix=''): """ result = "" if self._phase_statuses: - for phase, data in self._phase_statuses.iteritems(): + for phase, data in self._phase_statuses.items(): status, comments = data if not comments: result += "{}{} {} {}\n".format(prefix, status, self._test_name, phase) @@ -314,7 +314,7 @@ def get_overall_test_status(self, wait_for_run=False, check_throughput=False, ch """ rv = TEST_PASS_STATUS run_phase_found = False - for phase, data in self._phase_statuses.iteritems(): + for phase, data in self._phase_statuses.items(): status = data[0] if phase == RUN_PHASE: run_phase_found = True diff --git a/scripts/lib/CIME/user_mod_support.py b/scripts/lib/CIME/user_mod_support.py index 6c3ef34071b..d9eac7e7abb 100644 --- a/scripts/lib/CIME/user_mod_support.py +++ b/scripts/lib/CIME/user_mod_support.py @@ -92,7 +92,7 @@ def apply_user_mods(caseroot, user_mods_path, keepexe=None): for shell_command_file in case_shell_command_files: if os.path.isfile(shell_command_file): - os.chmod(shell_command_file, 0777) + os.chmod(shell_command_file, 0o777) run_cmd_no_fail(shell_command_file) diff --git a/scripts/lib/CIME/utils.py b/scripts/lib/CIME/utils.py index 6fbef55c61b..76b384a94b8 100644 --- a/scripts/lib/CIME/utils.py +++ b/scripts/lib/CIME/utils.py @@ -86,7 +86,7 @@ def _read_cime_config_file(): CIME_MODEL=acme,cesm PROJECT=someprojectnumber """ - from ConfigParser import SafeConfigParser as config_parser + from configparser import SafeConfigParser as config_parser cime_config_file = os.path.abspath(os.path.join(os.path.expanduser("~"), ".cime","config")) @@ -944,7 +944,7 @@ def compute_total_time(job_cost_map, proc_pool): running_jobs = {} # name -> (procs, est-time, start-time) while len(waiting_jobs) > 0 or len(running_jobs) > 0: launched_jobs = [] - for jobname, data in waiting_jobs.iteritems(): + for jobname, data in waiting_jobs.items(): procs_for_job, time_for_job = data if procs_for_job <= proc_pool: proc_pool -= procs_for_job @@ -955,7 +955,7 @@ def compute_total_time(job_cost_map, proc_pool): del waiting_jobs[launched_job] completed_jobs = [] - for jobname, data in running_jobs.iteritems(): + for jobname, data in running_jobs.items(): procs_for_job, time_for_job, time_started = data if (current_time - time_started) >= time_for_job: proc_pool += procs_for_job diff --git a/scripts/lib/CIME/wait_for_tests.py b/scripts/lib/CIME/wait_for_tests.py index a9b1d5ca6ef..85f6265db5f 100644 --- a/scripts/lib/CIME/wait_for_tests.py +++ b/scripts/lib/CIME/wait_for_tests.py @@ -1,4 +1,4 @@ -import os, time, threading, Queue, socket, signal, distutils.spawn, shutil, glob +import os, time, threading, queue, socket, signal, distutils.spawn, shutil, glob import logging import xml.etree.ElementTree as xmlet @@ -146,7 +146,7 @@ def create_cdash_upload_xml(results, cdash_build_name, cdash_build_group, utc_ti need_to_upload = False - for test_name, test_data in results.iteritems(): + for test_name, test_data in results.items(): test_path, test_status = test_data if (test_status not in [TEST_PASS_STATUS, NAMELIST_FAIL_STATUS]): @@ -299,7 +299,7 @@ def wait_for_test(test_path, results, wait, check_throughput, check_memory, igno ############################################################################### def wait_for_tests_impl(test_paths, no_wait=False, check_throughput=False, check_memory=False, ignore_namelists=False, ignore_memleak=False): ############################################################################### - results = Queue.Queue() + results = queue.Queue() for test_path in test_paths: t = threading.Thread(target=wait_for_test, args=(test_path, results, not no_wait, check_throughput, check_memory, ignore_namelists, ignore_memleak)) @@ -348,7 +348,7 @@ def wait_for_tests(test_paths, test_results = wait_for_tests_impl(test_paths, no_wait, check_throughput, check_memory, ignore_namelists, ignore_memleak) all_pass = True - for test_name, test_data in sorted(test_results.iteritems()): + for test_name, test_data in sorted(test_results.items()): test_path, test_status = test_data logging.info("Test '{}' finished with status '{}'".format(test_name, test_status)) logging.info(" Path: {}".format(test_path)) From ed893497e376e01976b2fb0a3b61b8a0df2f1941 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 3 Oct 2017 14:12:00 -0600 Subject: [PATCH 05/41] ran 2to3 on lib directory --- scripts/lib/jenkins_generic_job.py | 2 +- scripts/lib/update_acme_tests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/lib/jenkins_generic_job.py b/scripts/lib/jenkins_generic_job.py index c575cd74669..089bb7801b4 100644 --- a/scripts/lib/jenkins_generic_job.py +++ b/scripts/lib/jenkins_generic_job.py @@ -15,7 +15,7 @@ def cleanup_queue(test_root, test_id): with Case(case_dir, read_only=True) as case: jobmap = case.get_job_info() jobkills = [] - for jobname, jobid in jobmap.iteritems(): + for jobname, jobid in jobmap.items(): logging.warning("Found leftover batch job {} ({}) that need to be deleted".format(jobid, jobname)) jobkills.append(jobid) diff --git a/scripts/lib/update_acme_tests.py b/scripts/lib/update_acme_tests.py index 608c00aa422..c539cb01bdc 100644 --- a/scripts/lib/update_acme_tests.py +++ b/scripts/lib/update_acme_tests.py @@ -216,7 +216,7 @@ def get_test_suite(suite, machine=None, compiler=None): ############################################################################### def get_test_suites(): ############################################################################### - return _TEST_SUITES.keys() + return list(_TEST_SUITES.keys()) ############################################################################### def infer_machine_name_from_tests(testargs): From a4cd19e249397b8a9b6604e977d872048be2b9da Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 3 Oct 2017 14:14:27 -0600 Subject: [PATCH 06/41] ran 2to3 on scripts directory --- scripts/manage_pes | 2 +- scripts/query_config | 28 ++++++++++++++-------------- scripts/query_testlists | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/scripts/manage_pes b/scripts/manage_pes index da00c465c56..c366fe0d550 100755 --- a/scripts/manage_pes +++ b/scripts/manage_pes @@ -250,7 +250,7 @@ class ManagePes(object): self.print_gridnodes([match]) logger.info(" The new values would be") self.print_gridnodes([newmatch]) - override = raw_input(" Do you want to override the match with" + override = input(" Do you want to override the match with" " your pe-layout [yes/no] (default is no)\n") if override.lower() != "y" and override.lower() != "yes": logger.info("Nothing done.") diff --git a/scripts/query_config b/scripts/query_config index 4b2fb3a6094..889c06e6d9c 100755 --- a/scripts/query_config +++ b/scripts/query_config @@ -92,7 +92,7 @@ def print_compset(name, files, all_components=False): elif config_file is None or not os.path.isfile(config_file): return - print "\nActive component: {}".format(name) + print("\nActive component: {}".format(name)) # Now parse the compsets file and write out the compset alias and longname as well as the help text # determine component xml content compsets = Compsets(config_file) @@ -144,7 +144,7 @@ def query_component(name, all_components=False): expect(config_exists, "Cannot find config_file {} on disk".format(config_file)) elif all_components and not config_exists: - print "WARNING: Couldn't find config_file {} on disk".format(config_file) + print("WARNING: Couldn't find config_file {} on disk".format(config_file)) return # If name is not a valid argument - exit with error expect(match_found, @@ -285,10 +285,10 @@ class Machines(CIME.XML.machines.Machines): if single_machine and machine_name is None: files = Files() config_file = files.get_value("MACHINES_SPEC_FILE") - print "Machine is not listed in config file: {}".format(config_file) + print("Machine is not listed in config file: {}".format(config_file)) else: # write out machines machines = self.get_nodes(nodename="machine") - print "Machine(s)" + print("Machine(s)") for machine in machines: name = machine.get("MACH") desc = machine.find("DESC") @@ -300,21 +300,21 @@ class Machines(CIME.XML.machines.Machines): current_machine = self.probe_machine_name(warn=False) if not single_machine: name += " (current)" if current_machine and current_machine in name else "" - print " {} : {} ".format(name, desc.text) - print " os ", os_.text - print " compilers ",compilers.text + print(" {} : {} ".format(name, desc.text)) + print(" os ", os_.text) + print(" compilers ",compilers.text) if MAX_MPITASKS_PER_NODE is not None: - print " pes/node ",MAX_MPITASKS_PER_NODE.text + print(" pes/node ",MAX_MPITASKS_PER_NODE.text) if max_tasks_per_node is not None: - print " max_tasks/node ",max_tasks_per_node.text + print(" max_tasks/node ",max_tasks_per_node.text) elif single_machine and machine_name in name: - print " {} : {} ".format(name, desc.text) - print " os ", os_.text - print " compilers ",compilers.text + print(" {} : {} ".format(name, desc.text)) + print(" os ", os_.text) + print(" compilers ",compilers.text) if MAX_MPITASKS_PER_NODE is not None: - print " pes/node ",MAX_MPITASKS_PER_NODE.text + print(" pes/node ",MAX_MPITASKS_PER_NODE.text) if max_tasks_per_node is not None: - print " max_tasks/node ",max_tasks_per_node.text + print(" max_tasks/node ",max_tasks_per_node.text) def _main_func(description): """ diff --git a/scripts/query_testlists b/scripts/query_testlists index d363df3f564..3f314d868fb 100755 --- a/scripts/query_testlists +++ b/scripts/query_testlists @@ -13,7 +13,7 @@ Usage: All of the above support the various --xml-* arguments for subsetting which tests are included. """ -from __future__ import print_function + from Tools.standard_script_setup import * from CIME.test_utils import get_tests_from_xml, test_to_string from CIME.XML.tests import Tests From 97aa5a17a2421b06a966e6fb4b19d87c9b751aa7 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 3 Oct 2017 14:19:51 -0600 Subject: [PATCH 07/41] 2to3 on Tools --- scripts/Tools/case.qstatus | 2 +- scripts/Tools/case.submit | 2 +- scripts/Tools/check_case | 2 +- scripts/Tools/check_input_data | 2 +- scripts/Tools/check_lockedfiles | 2 +- scripts/Tools/cime_bisect | 10 ++++---- scripts/Tools/cimeteststatus | 22 ++++++++--------- scripts/Tools/code_checker | 4 +-- scripts/Tools/compare_namelists | 4 +-- scripts/Tools/compare_test_results | 2 +- scripts/Tools/component_compare_baseline | 4 +-- scripts/Tools/component_compare_copy | 2 +- scripts/Tools/component_compare_test | 4 +-- scripts/Tools/component_generate_baseline | 4 +-- scripts/Tools/cs.status | 6 ++--- scripts/Tools/getTiming | 2 +- scripts/Tools/jenkins_generic_job | 2 +- scripts/Tools/list_acme_tests | 6 ++--- scripts/Tools/normalize_cases | 2 +- scripts/Tools/pelayout | 6 ++--- scripts/Tools/preview_namelists | 2 +- scripts/Tools/preview_run | 10 ++++---- scripts/Tools/save_provenance | 2 +- scripts/Tools/simple_compare | 4 +-- scripts/Tools/testreporter.py | 2 +- scripts/Tools/update_acme_tests | 2 +- scripts/Tools/wait_for_tests | 2 +- scripts/Tools/xmlchange | 2 +- scripts/Tools/xmlquery | 30 +++++++++++------------ 29 files changed, 73 insertions(+), 73 deletions(-) diff --git a/scripts/Tools/case.qstatus b/scripts/Tools/case.qstatus index 9d64209e14e..748addf7e90 100755 --- a/scripts/Tools/case.qstatus +++ b/scripts/Tools/case.qstatus @@ -4,7 +4,7 @@ Show the batch status of all jobs associated with this case. """ -from standard_script_setup import * +from .standard_script_setup import * from CIME.case import Case from CIME.test_status import * diff --git a/scripts/Tools/case.submit b/scripts/Tools/case.submit index fc1787680bf..71c69cd4777 100755 --- a/scripts/Tools/case.submit +++ b/scripts/Tools/case.submit @@ -4,7 +4,7 @@ case.submit - Submit a cesm workflow to the queueing system or run it if there is no queueing system. A cesm workflow may include multiple jobs. """ -from standard_script_setup import * +from .standard_script_setup import * from CIME.case_submit import submit from CIME.case import Case diff --git a/scripts/Tools/check_case b/scripts/Tools/check_case index 07968a75ecb..c32cf928e9c 100755 --- a/scripts/Tools/check_case +++ b/scripts/Tools/check_case @@ -4,7 +4,7 @@ check_case verifies that the case is set up correctly """ -from standard_script_setup import * +from .standard_script_setup import * from CIME.utils import expect from CIME.case import Case diff --git a/scripts/Tools/check_input_data b/scripts/Tools/check_input_data index 811920ade56..78cf183ac78 100755 --- a/scripts/Tools/check_input_data +++ b/scripts/Tools/check_input_data @@ -5,7 +5,7 @@ Check (and optionally download) input data files. Should be run from case. """ -from standard_script_setup import * +from .standard_script_setup import * from CIME.utils import get_model from CIME.check_input_data import check_input_data, SVN_LOCS diff --git a/scripts/Tools/check_lockedfiles b/scripts/Tools/check_lockedfiles index 852ec72c20e..c327bf9b558 100755 --- a/scripts/Tools/check_lockedfiles +++ b/scripts/Tools/check_lockedfiles @@ -3,7 +3,7 @@ This script compares xml files """ -from standard_script_setup import * +from .standard_script_setup import * from CIME.check_lockedfiles import check_lockedfiles from CIME.case import Case diff --git a/scripts/Tools/cime_bisect b/scripts/Tools/cime_bisect index da144df52f1..e8871e51de7 100755 --- a/scripts/Tools/cime_bisect +++ b/scripts/Tools/cime_bisect @@ -5,7 +5,7 @@ A script to help track down the commit that caused a test to fail. This script is intended to be run by git bisect. """ -from standard_script_setup import * +from .standard_script_setup import * from CIME.utils import expect, run_cmd_no_fail, run_cmd from CIME.XML.machines import Machines @@ -95,9 +95,9 @@ def cime_bisect(testargs, good, bad, testroot, compiler, project, baseline_name, commits_we_want_to_test = run_cmd_no_fail("git rev-list {}..{} --merges --first-parent".format(good, bad)).splitlines() all_commits_ = run_cmd_no_fail("git rev-list {}..{}".format(good, bad)).splitlines() commits_to_skip = set(all_commits_) - set(commits_we_want_to_test) - print "Skipping these non-merge commits" + print("Skipping these non-merge commits") for item in commits_to_skip: - print item + print(item) else: commits_to_skip = set() @@ -145,8 +145,8 @@ def cime_bisect(testargs, good, bad, testroot, compiler, project, baseline_name, bad_commits_filtered = bad_commits - commits_to_skip expect(len(bad_commits_filtered) == 1, bad_commits_filtered) - print "Bad merge is:" - print run_cmd_no_fail("git show {}".format(bad_commits_filtered.pop())) + print("Bad merge is:") + print(run_cmd_no_fail("git show {}".format(bad_commits_filtered.pop()))) finally: run_cmd_no_fail("git bisect reset") diff --git a/scripts/Tools/cimeteststatus b/scripts/Tools/cimeteststatus index 7c8380ffc6a..7795a01e255 100755 --- a/scripts/Tools/cimeteststatus +++ b/scripts/Tools/cimeteststatus @@ -6,7 +6,7 @@ and send the test results back to the test database. Authors: Jay Shollenberger and Ben Andre """ -from __future__ import print_function + import sys if sys.hexversion < 0x02070000: print(70 * "*") @@ -18,8 +18,8 @@ if sys.hexversion < 0x02070000: import xml.etree.ElementTree as etree import argparse import os, glob, re -import urllib -import urllib2 +import urllib.request, urllib.parse, urllib.error +import urllib.request, urllib.error, urllib.parse import pprint import getpass @@ -221,11 +221,11 @@ class CimeTestStatus(): root = etree.Element('test') root.set('name', self._test_status['name']) root.set('status', self._test_status['status']) - for name, section in self._test_status.items(): + for name, section in list(self._test_status.items()): if isinstance(section, dict): elem = etree.Element('section') elem.set('name', name) - for key, value in section.items(): + for key, value in list(section.items()): child = etree.Element("{0}".format('key')) child.set('name', key) child.text = value @@ -354,10 +354,10 @@ def getSuiteInfo(specfile): for t in root.findall('test'): testlist.append(t.get('case')) - if 'machine' not in suiteinfo.keys(): + if 'machine' not in list(suiteinfo.keys()): machnodelist = t.findall('mach') suiteinfo['machine'] = machnodelist[0].text - if 'compiler' not in suiteinfo.keys(): + if 'compiler' not in list(suiteinfo.keys()): compnodelist = t.findall('compiler') suiteinfo['compiler'] = compnodelist[0].text @@ -527,13 +527,13 @@ def sendTestReport(args, suiteinfo, cimetests, auth): # Get the testdb username/password, and POST # the data. print("sending test report for " + suiteinfo['machine'] + " " + suiteinfo['compiler']) - data = urllib.urlencode({'username':auth['username'], + data = urllib.parse.urlencode({'username':auth['username'], 'password':auth['password'], 'testXML':testrecord}) - req = urllib2.Request(testdburl, data) + req = urllib.request.Request(testdburl, data) try: - urllib2.urlopen(req) - except urllib2.URLError as e: + urllib.request.urlopen(req) + except urllib.error.URLError as e: print("Error when posting data: " + e.reason) if(args.debug): diff --git a/scripts/Tools/code_checker b/scripts/Tools/code_checker index 63c04e88139..f57dfa2ab45 100755 --- a/scripts/Tools/code_checker +++ b/scripts/Tools/code_checker @@ -5,7 +5,7 @@ Ensure that all CIME python files are free of errors and follow the PEP8 standard. """ -from standard_script_setup import * +from .standard_script_setup import * from CIME.code_checker import check_code, expect @@ -63,7 +63,7 @@ def _main_func(description): num_procs, files = parse_command_line(sys.argv, description) results = check_code(files, num_procs=num_procs, interactive=True) - for result in results.itervalues(): + for result in results.values(): if result != "": sys.exit(1) diff --git a/scripts/Tools/compare_namelists b/scripts/Tools/compare_namelists index 56d6eb52eea..f8b99b5ef06 100755 --- a/scripts/Tools/compare_namelists +++ b/scripts/Tools/compare_namelists @@ -5,7 +5,7 @@ Compare namelists. Should be called by an ACME test. Designed to not be sensitive to order or whitespace. """ -from standard_script_setup import * +from .standard_script_setup import * import CIME.compare_namelists from CIME.utils import expect @@ -67,7 +67,7 @@ def _main_func(description): expect(success, "Namelist diff between files {} and {}\n{}".format(gold_file, compare_file, comments)) - print "Files {} and {} MATCH".format(gold_file, compare_file) + print("Files {} and {} MATCH".format(gold_file, compare_file)) ############################################################################### diff --git a/scripts/Tools/compare_test_results b/scripts/Tools/compare_test_results index 91967961a71..2f2e0579f45 100755 --- a/scripts/Tools/compare_test_results +++ b/scripts/Tools/compare_test_results @@ -21,7 +21,7 @@ You may need to load modules for cprnc to work. """ -from standard_script_setup import * +from .standard_script_setup import * from CIME.XML.machines import Machines from CIME.compare_test_results import compare_test_results diff --git a/scripts/Tools/component_compare_baseline b/scripts/Tools/component_compare_baseline index b43b08c8caf..46a6bf36f9d 100755 --- a/scripts/Tools/component_compare_baseline +++ b/scripts/Tools/component_compare_baseline @@ -4,7 +4,7 @@ Compares current component history files against baselines """ -from standard_script_setup import * +from .standard_script_setup import * from CIME.case import Case from CIME.hist_utils import compare_baseline @@ -45,7 +45,7 @@ def _main_func(description): caseroot, baseline_dir = parse_command_line(sys.argv, description) with Case(caseroot) as case: success, comments = compare_baseline(case, baseline_dir) - print comments + print(comments) sys.exit(0 if success else 1) diff --git a/scripts/Tools/component_compare_copy b/scripts/Tools/component_compare_copy index d7c47b09b96..0ccbd975c7a 100755 --- a/scripts/Tools/component_compare_copy +++ b/scripts/Tools/component_compare_copy @@ -5,7 +5,7 @@ Copy the most recent batch of hist files in a case, adding the given suffix. This allows us to save these results if we want to run the case again. """ -from standard_script_setup import * +from .standard_script_setup import * from CIME.case import Case from CIME.hist_utils import copy diff --git a/scripts/Tools/component_compare_test b/scripts/Tools/component_compare_test index 8deace7958f..bfdb885919b 100755 --- a/scripts/Tools/component_compare_test +++ b/scripts/Tools/component_compare_test @@ -4,7 +4,7 @@ Compares two component history files in the testcase directory """ -from standard_script_setup import * +from .standard_script_setup import * from CIME.case import Case from CIME.hist_utils import compare_test @@ -48,7 +48,7 @@ def _main_func(description): suffix1, suffix2, caseroot = parse_command_line(sys.argv, description) with Case(caseroot) as case: success, comments = compare_test(case, suffix1, suffix2) - print comments + print(comments) sys.exit(0 if success else 1) diff --git a/scripts/Tools/component_generate_baseline b/scripts/Tools/component_generate_baseline index 9a549c3ee1d..547404a8a5f 100755 --- a/scripts/Tools/component_generate_baseline +++ b/scripts/Tools/component_generate_baseline @@ -4,7 +4,7 @@ Copies current component history files into baselines """ -from standard_script_setup import * +from .standard_script_setup import * from CIME.case import Case from CIME.hist_utils import generate_baseline @@ -52,7 +52,7 @@ def _main_func(description): with Case(caseroot) as case: success, comments = generate_baseline(case, baseline_dir, allow_baseline_overwrite) - print comments + print(comments) sys.exit(0 if success else 1) diff --git a/scripts/Tools/cs.status b/scripts/Tools/cs.status index e75234b4fc7..38799631193 100755 --- a/scripts/Tools/cs.status +++ b/scripts/Tools/cs.status @@ -4,7 +4,7 @@ List test results based on TestStatus files. Returns True if no errors occured (not based on test statuses). """ -from standard_script_setup import * +from .standard_script_setup import * import argparse, sys, os, logging, glob from CIME.test_status import * @@ -65,8 +65,8 @@ def cs_status(test_paths, summary=False): test_id_output[test_id] = output for test_id in sorted(test_id_output): - print test_id - print test_id_output[test_id], + print(test_id) + print(test_id_output[test_id], end=' ') ############################################################################### def _main_func(description): diff --git a/scripts/Tools/getTiming b/scripts/Tools/getTiming index 22878582ee5..a3efa3e7ad3 100755 --- a/scripts/Tools/getTiming +++ b/scripts/Tools/getTiming @@ -3,7 +3,7 @@ Get timing information from run """ -from standard_script_setup import * +from .standard_script_setup import * import argparse, sys, os from CIME.case import Case from CIME.get_timing import get_timing diff --git a/scripts/Tools/jenkins_generic_job b/scripts/Tools/jenkins_generic_job index 1331a4dbced..7870ecf5ae8 100755 --- a/scripts/Tools/jenkins_generic_job +++ b/scripts/Tools/jenkins_generic_job @@ -7,7 +7,7 @@ wait_for_tests that handles cleanup of old test results and ensures that the batch system is left in a clean state. """ -from standard_script_setup import * +from .standard_script_setup import * import CIME.wait_for_tests from CIME.utils import expect diff --git a/scripts/Tools/list_acme_tests b/scripts/Tools/list_acme_tests index 102cbb7d50e..46f25f21510 100755 --- a/scripts/Tools/list_acme_tests +++ b/scripts/Tools/list_acme_tests @@ -5,7 +5,7 @@ List acme test suites. Can be used to show what's being tested. Can just list tested grids, compsets, etc. """ -from standard_script_setup import * +from .standard_script_setup import * from CIME.utils import expect import update_acme_tests @@ -69,9 +69,9 @@ def list_tests(thing_to_list, categories): else: expect(False, "Unrecognized thing to list '{}'".format(thing_to_list)) - print("Tested {} for test categories: {}".format(thing_to_list, ", ".join(categories))) + print(("Tested {} for test categories: {}".format(thing_to_list, ", ".join(categories)))) for item in sorted(things): - print(" ", item) + print((" ", item)) ############################################################################### def _main_func(description): diff --git a/scripts/Tools/normalize_cases b/scripts/Tools/normalize_cases index 848077ee731..68395807b9e 100755 --- a/scripts/Tools/normalize_cases +++ b/scripts/Tools/normalize_cases @@ -8,7 +8,7 @@ This is for debugging purposes and meant to assist the user when they want to run case_diff. """ -from standard_script_setup import * +from .standard_script_setup import * from CIME.utils import expect, run_cmd_no_fail import argparse, sys, os, doctest, glob diff --git a/scripts/Tools/pelayout b/scripts/Tools/pelayout index 7effc06288c..abb60f04168 100755 --- a/scripts/Tools/pelayout +++ b/scripts/Tools/pelayout @@ -28,7 +28,7 @@ information is available at https://github.com/ESMCI/cime. it under the same terms as Python itself. """ -from standard_script_setup import * +from .standard_script_setup import * from CIME.case import Case from CIME.utils import expect, convert_to_string @@ -113,10 +113,10 @@ def print_pelayout(case, ntasks, nthreads, rootpes, arg_format, header): comp_classes = case.get_values("COMP_CLASSES") if (header is not None): - print header + print(header) # End if for comp in comp_classes: - print format_pelayout(comp, ntasks[comp], nthreads[comp], rootpes[comp], arg_format) + print(format_pelayout(comp, ntasks[comp], nthreads[comp], rootpes[comp], arg_format)) # End for # End def print_pelayout diff --git a/scripts/Tools/preview_namelists b/scripts/Tools/preview_namelists index 4bece0ee573..491c47d94a3 100755 --- a/scripts/Tools/preview_namelists +++ b/scripts/Tools/preview_namelists @@ -6,7 +6,7 @@ which creates the namelist and other model input files, it then copies those files to the CaseDocs subdirectory for inspection """ -from standard_script_setup import * +from .standard_script_setup import * from CIME.preview_namelists import create_namelists from CIME.case import Case diff --git a/scripts/Tools/preview_run b/scripts/Tools/preview_run index a64d07a5172..91deb961376 100755 --- a/scripts/Tools/preview_run +++ b/scripts/Tools/preview_run @@ -16,7 +16,7 @@ Example: ./xmlchange --subgroup=case.run BATCH_COMMAND_FLAGS='--some-flag --other-flag' """ -from standard_script_setup import * +from .standard_script_setup import * from CIME.case import Case @@ -57,13 +57,13 @@ def _main_func(description): logging.disable(logging.CRITICAL) with Case(caseroot, read_only=False) as case: - print "BATCH SUBMIT:" + print("BATCH SUBMIT:") job = "case.test" if case.get_value("TEST") else "case.run" job_id_to_cmd = case.submit_jobs(dry_run=True, job=job) for job_id, cmd in job_id_to_cmd: - print " ", job_id, "->", case.get_resolved_value(cmd) - print - print "MPIRUN:", case.get_resolved_value(case.get_mpirun_cmd()) + print(" ", job_id, "->", case.get_resolved_value(cmd)) + print() + print("MPIRUN:", case.get_resolved_value(case.get_mpirun_cmd())) if __name__ == "__main__": _main_func(__doc__) diff --git a/scripts/Tools/save_provenance b/scripts/Tools/save_provenance index ce68a4d5ac2..039867c7061 100755 --- a/scripts/Tools/save_provenance +++ b/scripts/Tools/save_provenance @@ -4,7 +4,7 @@ This tool provide command-line access to provenance-saving functionality """ -from standard_script_setup import * +from .standard_script_setup import * from CIME.case import Case from CIME.provenance import * diff --git a/scripts/Tools/simple_compare b/scripts/Tools/simple_compare index 26c438ce82c..99767bf8e6f 100755 --- a/scripts/Tools/simple_compare +++ b/scripts/Tools/simple_compare @@ -5,7 +5,7 @@ Compare files in a normalized way. Used by create_test for diffing non-namelist files. """ -from standard_script_setup import * +from .standard_script_setup import * import CIME.simple_compare from CIME.utils import expect @@ -67,7 +67,7 @@ def _main_func(description): expect(success, "Diff between files {} and {}:\n{}".format(gold_file, compare_file, comments)) - print("Files {} and {} MATCH".format(gold_file, compare_file)) + print(("Files {} and {} MATCH".format(gold_file, compare_file))) ############################################################################### diff --git a/scripts/Tools/testreporter.py b/scripts/Tools/testreporter.py index 41f73bf0372..2f87cf97056 100755 --- a/scripts/Tools/testreporter.py +++ b/scripts/Tools/testreporter.py @@ -6,7 +6,7 @@ """ -from standard_script_setup import * +from .standard_script_setup import * from CIME.XML.env_build import EnvBuild from CIME.XML.env_case import EnvCase diff --git a/scripts/Tools/update_acme_tests b/scripts/Tools/update_acme_tests index 4a6cdaec141..38c32ee5ccc 100755 --- a/scripts/Tools/update_acme_tests +++ b/scripts/Tools/update_acme_tests @@ -8,7 +8,7 @@ this file to create new test entries, and then inserts these entries into the XML file. """ -from standard_script_setup import * +from .standard_script_setup import * from CIME.utils import expect from update_acme_tests import get_test_suites, update_acme_tests diff --git a/scripts/Tools/wait_for_tests b/scripts/Tools/wait_for_tests index fa97f818c16..471126979e4 100755 --- a/scripts/Tools/wait_for_tests +++ b/scripts/Tools/wait_for_tests @@ -8,7 +8,7 @@ for the RUN phase specifically and will not terminate if the RUN phase didn't happen. """ -from standard_script_setup import * +from .standard_script_setup import * import CIME.wait_for_tests diff --git a/scripts/Tools/xmlchange b/scripts/Tools/xmlchange index 55d4b897d2c..fdfdb712896 100755 --- a/scripts/Tools/xmlchange +++ b/scripts/Tools/xmlchange @@ -10,7 +10,7 @@ """ -from standard_script_setup import * +from .standard_script_setup import * from CIME.utils import expect, convert_to_type, append_case_status from CIME.case import Case diff --git a/scripts/Tools/xmlquery b/scripts/Tools/xmlquery index 93239b4076f..8bc9ecd33c2 100755 --- a/scripts/Tools/xmlquery +++ b/scripts/Tools/xmlquery @@ -17,7 +17,7 @@ If this tool is missing any feature that you need, please go to github and creat it under the same terms as Python itself. """ -from standard_script_setup import * +from .standard_script_setup import * from CIME.case import Case from CIME.utils import expect, convert_to_string @@ -327,38 +327,38 @@ def _main_func(): wrapper.fix_sentence_endings = True for group in sorted(iter(results)): - if (len(variables) > 1 or len(results.keys()) > 1 or full) and not get_group: - print "\nResults in group %s"%group + if (len(variables) > 1 or len(list(results.keys())) > 1 or full) and not get_group: + print("\nResults in group %s"%group) for var in variables: if var in results[group]: if raw: - print results[group][var]['raw'] + print(results[group][var]['raw']) elif get_group: - print "\t%s: %s"%(var, results[group][var]['get_group']) + print("\t%s: %s"%(var, results[group][var]['get_group'])) elif value: sys.stdout.write("%s"%results[group][var]['value']) elif description: if results[group][var]['desc'][0] is not None: desc_text = ' '.join(results[group][var]['desc'][0].split()) - print "\t%s: %s"%(var, wrapper.fill(desc_text)) + print("\t%s: %s"%(var, wrapper.fill(desc_text))) elif fileonly: - print "\t%s: %s"%(var, results[group][var]['file']) + print("\t%s: %s"%(var, results[group][var]['file'])) elif dtype: - print "\t%s: %s"%(var, results[group][var]['type']) + print("\t%s: %s"%(var, results[group][var]['type'])) elif valid_values: if 'valid_values' in results[group][var]: - print "\t%s: %s"%(var, results[group][var]["valid_values"]) + print("\t%s: %s"%(var, results[group][var]["valid_values"])) elif full: if results[group][var]['desc'][0] is not None: desc_text = ' '.join(results[group][var]['desc'][0].split()) - print "\t%s: value=%s"%(var, results[group][var]['value']) - print "\t\ttype: %s"%(results[group][var]['type'][0]) + print("\t%s: value=%s"%(var, results[group][var]['value'])) + print("\t\ttype: %s"%(results[group][var]['type'][0])) if 'valid_values' in results[group][var]: - print "\t\tvalid_values: %s"%(results[group][var]["valid_values"]) - print "\t\tdescription: %s"%(wrapper.fill(desc_text)) - print "\t\tfile: %s"%(results[group][var]['file'][0]) + print("\t\tvalid_values: %s"%(results[group][var]["valid_values"])) + print("\t\tdescription: %s"%(wrapper.fill(desc_text))) + print("\t\tfile: %s"%(results[group][var]['file'][0])) else: - print "\t%s: %s"%(var, results[group][var]['value']) + print("\t%s: %s"%(var, results[group][var]['value'])) if (__name__ == "__main__"): From 06b4a7f201fa8eb7008bd7e2e1e9a548dc76b9db Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 3 Oct 2017 14:20:52 -0600 Subject: [PATCH 08/41] 2to3 on tests --- scripts/tests/list_tests | 4 ++-- scripts/tests/scripts_regression_tests.py | 22 +++++++++++----------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/scripts/tests/list_tests b/scripts/tests/list_tests index d9de526aa85..4f869e7c143 100755 --- a/scripts/tests/list_tests +++ b/scripts/tests/list_tests @@ -13,14 +13,14 @@ def list_tests_from(): if len(tests): for atest in tests: if DEBUG: - print atest + print(atest) for btest in atest._tests: btestname = btest.__str__().split() test_classes.append(btestname[1][1:-1].split('.')[1]) # add this explicitly, not captured by the above test_classes.append("B_CheckCode") for ctest in sorted(list(set(test_classes))): - print ctest + print(ctest) if __name__ == "__main__": # Include the directories diff --git a/scripts/tests/scripts_regression_tests.py b/scripts/tests/scripts_regression_tests.py index 7382a0e2c25..7ef9bb14a74 100755 --- a/scripts/tests/scripts_regression_tests.py +++ b/scripts/tests/scripts_regression_tests.py @@ -239,8 +239,8 @@ def tearDownClass(cls): teardown_root = True for tfile in cls._testdirs: if tfile not in cls._do_teardown: - print "Detected failed test or user request no teardown" - print "Leaving case directory : %s"%tfile + print("Detected failed test or user request no teardown") + print("Leaving case directory : %s"%tfile) teardown_root = False elif do_teardown: shutil.rmtree(tfile) @@ -517,8 +517,8 @@ def tearDownClass(cls): for tfile in cls._testdirs: if tfile not in cls._do_teardown: - print "Detected failed test or user request no teardown" - print "Leaving case directory : %s"%tfile + print("Detected failed test or user request no teardown") + print("Leaving case directory : %s"%tfile) elif do_teardown: shutil.rmtree(tfile) @@ -768,10 +768,10 @@ def tearDown(self): do_teardown = self._do_teardown and sys.exc_info() == (None, None, None) if (not do_teardown): - print "Detected failed test or user request no teardown" - print "Leaving files:" + print("Detected failed test or user request no teardown") + print("Leaving files:") for file_to_clean in files_to_clean: - print " ", file_to_clean + print(" ", file_to_clean) else: # For batch machines need to avoid race condition as batch system # finishes I/O for the case. @@ -1692,7 +1692,7 @@ def get_macros(macro_maker, build_xml, build_system): """ # Build.write_macros expects file-like objects as input, so # we need to wrap the strings in StringIO objects. - xml = io.StringIO(unicode(build_xml)) + xml = io.StringIO(str(build_xml)) output = io.StringIO() output_format = None if build_system == "Makefile": @@ -2337,7 +2337,7 @@ def check_for_pylint(): major = int(pylintver.group(1)) minor = int(pylintver.group(2)) if pylint is None or major < 1 or (major == 1 and minor < 5): - print "pylint version 1.5 or newer not found, pylint tests skipped" + print("pylint version 1.5 or newer not found, pylint tests skipped") return False return True @@ -2436,9 +2436,9 @@ def _main_func(): except SystemExit: had_fails = sys.exc_info()[1].message if had_fails: - print "Detected failures, leaving directory:", TEST_ROOT + print("Detected failures, leaving directory:", TEST_ROOT) else: - print "All pass, removing directory:", TEST_ROOT + print("All pass, removing directory:", TEST_ROOT) if os.path.exists(TEST_ROOT): shutil.rmtree(TEST_ROOT) From ff48e259d21ac9ebe4e73b7ececb2dff2b967825 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 3 Oct 2017 16:16:03 -0600 Subject: [PATCH 09/41] passes A_RunUnitTests --- .../lib/CIME/BuildTools/macrowriterbase.py | 7 +- scripts/lib/CIME/XML/test_reporter.py | 6 +- scripts/lib/CIME/namelist.py | 294 +++++++++--------- scripts/lib/CIME/utils.py | 2 +- scripts/lib/CIME/wait_for_tests.py | 4 +- scripts/tests/scripts_regression_tests.py | 2 +- 6 files changed, 158 insertions(+), 157 deletions(-) diff --git a/scripts/lib/CIME/BuildTools/macrowriterbase.py b/scripts/lib/CIME/BuildTools/macrowriterbase.py index c0d33fc9071..8061d6414a2 100644 --- a/scripts/lib/CIME/BuildTools/macrowriterbase.py +++ b/scripts/lib/CIME/BuildTools/macrowriterbase.py @@ -14,6 +14,8 @@ from abc import ABCMeta, abstractmethod from CIME.XML.standard_module_setup import * from CIME.utils import get_cime_root +from CIME.six import add_metaclass + logger = logging.getLogger(__name__) def _get_components(value): @@ -69,7 +71,8 @@ def _get_components(value): return components -class MacroWriterBase(object, metaclass=ABCMeta): +@add_metaclass(ABCMeta) +class MacroWriterBase(object): """Abstract base class for macro file writers. @@ -130,7 +133,7 @@ def write_line(self, line): A trailing newline is added, whether or not the input has one. """ - self.output.write(str(self.indent_string() + line + "\n")) + self.output.write(unicode(self.indent_string() + line + "\n")) @abstractmethod def environment_variable_string(self, name): diff --git a/scripts/lib/CIME/XML/test_reporter.py b/scripts/lib/CIME/XML/test_reporter.py index ca63481c2ae..c0a5987c186 100644 --- a/scripts/lib/CIME/XML/test_reporter.py +++ b/scripts/lib/CIME/XML/test_reporter.py @@ -2,15 +2,11 @@ Interface to the testreporter xml. This class inherits from GenericXML.py """ - +from CIME.six.moves.urllib import request, parse, error from CIME.XML.standard_module_setup import * from CIME.XML.generic_xml import GenericXML from CIME.utils import expect,get_model -import urllib.request, urllib.parse, urllib.error - - - class TestReporter(GenericXML): def __init__(self): diff --git a/scripts/lib/CIME/namelist.py b/scripts/lib/CIME/namelist.py index 5e31a95d35b..43377028bf7 100644 --- a/scripts/lib/CIME/namelist.py +++ b/scripts/lib/CIME/namelist.py @@ -688,8 +688,8 @@ def expand_literal_list(literals): ['true'] >>> expand_literal_list(['1', '2', 'f*', '3*3', '5']) ['1', '2', 'f*', '3', '3', '3', '5'] - >>> expand_literal_list([u'2*f*']) - [u'f*', u'f*'] + >>> expand_literal_list(['2*f*']) + ['f*', 'f*'] """ expanded = [] for literal in literals: @@ -711,8 +711,8 @@ def compress_literal_list(literals): ['true'] >>> compress_literal_list(['1', '2', 'f*', '3', '3', '3', '5']) ['1', '2', 'f*', '3', '3', '3', '5'] - >>> compress_literal_list([u'f*', u'f*']) - [u'f*', u'f*'] + >>> compress_literal_list(['f*', 'f*']) + ['f*', 'f*'] """ compressed = [] if len(literals) == 0: @@ -899,7 +899,7 @@ def get_group_names(self): >>> Namelist().get_group_names() [] >>> sorted(parse(text='&foo / &bar /').get_group_names()) - [u'bar', u'foo'] + ['bar', 'foo'] """ return list(self._groups.keys()) @@ -912,13 +912,13 @@ def get_variable_names(self, group_name): [] >>> x = parse(text='&foo bar=,bazz=true,bazz(2)=fred,bang=6*""/') >>> sorted(x.get_variable_names('fOo')) - [u'bang', u'bar', u'bazz', u'bazz(2)'] + ['bang', 'bar', 'bazz', 'bazz(2)'] >>> x = parse(text='&foo bar=,bazz=true,bang=6*""/') >>> sorted(x.get_variable_names('fOo')) - [u'bang', u'bar', u'bazz'] + ['bang', 'bar', 'bazz'] >>> x = parse(text='&foo bar(::)=,bazz=false,bazz(2)=true,bazz(:2:)=6*""/') >>> sorted(x.get_variable_names('fOo')) - [u'bar(::)', u'bazz', u'bazz(2)', u'bazz(:2:)'] + ['bar(::)', 'bazz', 'bazz(2)', 'bazz(:2:)'] """ group_name = group_name.lower() if group_name not in self._groups: @@ -929,15 +929,15 @@ def get_variable_value(self, group_name, variable_name): """Return the value of the specified variable. This function always returns a non-empty list containing strings. If the - specified `group_name` or `variable_name` is not present, `[u'']` is + specified `group_name` or `variable_name` is not present, `['']` is returned. >>> Namelist().get_variable_value('foo', 'bar') - [u''] + [''] >>> parse(text='&foo bar=1,2 /').get_variable_value('foo', 'bazz') - [u''] + [''] >>> parse(text='&foo bar=1,2 /').get_variable_value('foO', 'Bar') - [u'1', u'2'] + ['1', '2'] """ group_name = group_name.lower() variable_name = variable_name.lower() @@ -957,13 +957,13 @@ def get_value(self, variable_name): >>> parse(text='&foo bar=1 / &bazz bar=1 /').get_value('bar') Traceback (most recent call last): ... - SystemExit: ERROR: Namelist.get_value: Variable {} is present in multiple groups: [u'bazz', u'foo'] + SystemExit: ERROR: Namelist.get_value: Variable {} is present in multiple groups: ['bazz', 'foo'] >>> parse(text='&foo bar=1 / &bazz /').get_value('Bar') - [u'1'] + ['1'] >>> parse(text='&foo bar(2)=1 / &bazz /').get_value('Bar(2)') - [u'1'] + ['1'] >>> parse(text='&foo / &bazz /').get_value('bar') - [u''] + [''] """ variable_name = variable_name.lower() possible_groups = [group_name for group_name in self._groups @@ -981,25 +981,25 @@ def set_variable_value(self, group_name, variable_name, value, var_size=1): >>> x = parse(text='&foo bar=1 /') >>> x.get_variable_value('foo', 'bar') - [u'1'] - >>> x.set_variable_value('foo', 'bar(2)', [u'3'], var_size=4) + ['1'] + >>> x.set_variable_value('foo', 'bar(2)', ['3'], var_size=4) >>> x.get_variable_value('foo', 'bar') - [u'1', u'3'] - >>> x.set_variable_value('foo', 'bar(1)', [u'2']) + ['1', '3'] + >>> x.set_variable_value('foo', 'bar(1)', ['2']) >>> x.get_variable_value('foo', 'bar') - [u'2', u'3'] - >>> x.set_variable_value('foo', 'bar', [u'1']) + ['2', '3'] + >>> x.set_variable_value('foo', 'bar', ['1']) >>> x.get_variable_value('foo', 'bar') - [u'1', u'3'] - >>> x.set_variable_value('foo', 'bazz', [u'3']) - >>> x.set_variable_value('Brack', 'baR', [u'4']) + ['1', '3'] + >>> x.set_variable_value('foo', 'bazz', ['3']) + >>> x.set_variable_value('Brack', 'baR', ['4']) >>> x.get_variable_value('foo', 'bazz') - [u'3'] + ['3'] >>> x.get_variable_value('brack', 'bar') - [u'4'] - >>> x.set_variable_value('foo', 'red(2:6:2)', [u'2', u'4', u'6'], var_size=12) + ['4'] + >>> x.set_variable_value('foo', 'red(2:6:2)', ['2', '4', '6'], var_size=12) >>> x.get_variable_value('foo', 'red') - ['', u'2', '', u'4', '', u'6'] + ['', '2', '', '4', '', '6'] """ group_name = group_name.lower() @@ -1058,43 +1058,43 @@ def merge_nl(self, other, overwrite=False): >>> x = parse(text='&foo bar=1 bazz=,2 brat=3/') >>> y = parse(text='&foo bar=2 bazz=3*1 baker=4 / &foo2 barter=5 /') >>> y.get_value('bazz') - [u'1', u'1', u'1'] + ['1', '1', '1'] >>> x.merge_nl(y) >>> sorted(x.get_group_names()) - [u'foo', u'foo2'] + ['foo', 'foo2'] >>> sorted(x.get_variable_names('foo')) - [u'baker', u'bar', u'bazz', u'brat'] + ['baker', 'bar', 'bazz', 'brat'] >>> sorted(x.get_variable_names('foo2')) - [u'barter'] + ['barter'] >>> x.get_value('bar') - [u'1'] + ['1'] >>> x.get_value('bazz') - [u'1', u'2', u'1'] + ['1', '2', '1'] >>> x.get_value('brat') - [u'3'] + ['3'] >>> x.get_value('baker') - [u'4'] + ['4'] >>> x.get_value('barter') - [u'5'] + ['5'] >>> x = parse(text='&foo bar=1 bazz=,2 brat=3/') >>> y = parse(text='&foo bar=2 bazz=3*1 baker=4 / &foo2 barter=5 /') >>> x.merge_nl(y, overwrite=True) >>> sorted(x.get_group_names()) - [u'foo', u'foo2'] + ['foo', 'foo2'] >>> sorted(x.get_variable_names('foo')) - [u'baker', u'bar', u'bazz', u'brat'] + ['baker', 'bar', 'bazz', 'brat'] >>> sorted(x.get_variable_names('foo2')) - [u'barter'] + ['barter'] >>> x.get_value('bar') - [u'2'] + ['2'] >>> x.get_value('bazz') - [u'1', u'1', u'1'] + ['1', '1', '1'] >>> x.get_value('brat') - [u'3'] + ['3'] >>> x.get_value('baker') - [u'4'] + ['4'] >>> x.get_value('barter') - [u'5'] + ['5'] """ # Pretty simple strategy: go through the entire other namelist, and # merge all values with this one's. @@ -1331,7 +1331,7 @@ def _advance(self, nchars=1, check_eof=False): >>> x._advance(check_eof=True) False >>> x._curr() - u'b' + 'b' >>> x._advance(check_eof=True) True """ @@ -1365,7 +1365,7 @@ def _eat_whitespace(self, allow_initial_comment=False): >>> x._eat_whitespace() True >>> x._curr() - u'a' + 'a' >>> x._eat_whitespace() False >>> x._advance() @@ -1377,22 +1377,22 @@ def _eat_whitespace(self, allow_initial_comment=False): >>> x._eat_whitespace() True >>> x._curr() - u'a' + 'a' >>> x = _NamelistParser('! blah\n a') >>> x._eat_whitespace() False >>> x._curr() - u'!' + '!' >>> x = _NamelistParser(' ! blah\n a') >>> x._eat_whitespace() True >>> x._curr() - u'!' + '!' >>> x = _NamelistParser(' ! blah\n a') >>> x._eat_whitespace(allow_initial_comment=True) True >>> x._curr() - u'a' + 'a' """ eaten = False comment_allowed = allow_initial_comment @@ -1416,7 +1416,7 @@ def _eat_comment(self): >>> x._eat_comment() True >>> x._curr() - u' ' + ' ' >>> x._eat_comment() False >>> x._eat_whitespace() @@ -1424,7 +1424,7 @@ def _eat_comment(self): >>> x._eat_comment() True >>> x._curr() - u'a' + 'a' >>> x._advance(2) >>> x._eat_comment() Traceback (most recent call last): @@ -1483,9 +1483,9 @@ def _parse_namelist_group_name(self): ... _NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser('&abc ')._parse_namelist_group_name() - u'abc' + 'abc' >>> _NamelistParser('&abc\n')._parse_namelist_group_name() - u'abc' + 'abc' >>> _NamelistParser('&abc/ ')._parse_namelist_group_name() Traceback (most recent call last): ... @@ -1515,21 +1515,21 @@ def _parse_variable_name(self, allow_equals=True): ... _NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser('foo(2)= ')._parse_variable_name() - u'foo(2)' + 'foo(2)' >>> _NamelistParser('abc ')._parse_variable_name() - u'abc' + 'abc' >>> _NamelistParser('ABC ')._parse_variable_name() - u'abc' + 'abc' >>> _NamelistParser('abc\n')._parse_variable_name() - u'abc' + 'abc' >>> _NamelistParser('abc%fred\n')._parse_variable_name() - u'abc%fred' + 'abc%fred' >>> _NamelistParser('abc(2)@fred\n')._parse_variable_name() - u'abc(2)@fred' + 'abc(2)@fred' >>> _NamelistParser('abc(1:2:3)\n')._parse_variable_name() - u'abc(1:2:3)' + 'abc(1:2:3)' >>> _NamelistParser('abc=')._parse_variable_name() - u'abc' + 'abc' >>> _NamelistParser('abc(1,2) ')._parse_variable_name() Traceback (most recent call last): ... @@ -1543,7 +1543,7 @@ def _parse_variable_name(self, allow_equals=True): ... _NamelistParseError: Error in parsing namelist: '' is not a valid variable name >>> _NamelistParser('foo+= ')._parse_variable_name() - u'foo' + 'foo' """ old_pos = self._pos separators = (' ', '\n', '=', '+') if allow_equals else (' ', '\n') @@ -1586,19 +1586,19 @@ def _parse_character_literal(self): ... _NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser('"abc" ')._parse_character_literal() - u'"abc"' + '"abc"' >>> _NamelistParser("'abc' ")._parse_character_literal() - u"'abc'" + "'abc'" >>> _NamelistParser("*abc* ")._parse_character_literal() Traceback (most recent call last): ... _NamelistParseError: Error in parsing namelist: *abc* is not a valid character literal >>> _NamelistParser("'abc''def' ")._parse_character_literal() - u"'abc''def'" + "'abc''def'" >>> _NamelistParser("'abc''' ")._parse_character_literal() - u"'abc'''" + "'abc'''" >>> _NamelistParser("'''abc' ")._parse_character_literal() - u"'''abc'" + "'''abc'" """ delimiter = self._curr() old_pos = self._pos @@ -1630,7 +1630,7 @@ def _parse_complex_literal(self): ... _NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser('(1.,2.) ')._parse_complex_literal() - u'(1.,2.)' + '(1.,2.)' >>> _NamelistParser("(A,B) ")._parse_complex_literal() Traceback (most recent call last): ... @@ -1710,57 +1710,57 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): to mark the end of a literal. >>> _NamelistParser('"abc" ')._parse_literal() - u'"abc"' + '"abc"' >>> _NamelistParser("'abc' ")._parse_literal() - u"'abc'" + "'abc'" >>> _NamelistParser('"abc"')._parse_literal() Traceback (most recent call last): ... _NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser('"abc"')._parse_literal(allow_eof_end=True) - u'"abc"' + '"abc"' >>> _NamelistParser('(1.,2.) ')._parse_literal() - u'(1.,2.)' + '(1.,2.)' >>> _NamelistParser('(1.,2.)')._parse_literal() Traceback (most recent call last): ... _NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser('(1.,2.)')._parse_literal(allow_eof_end=True) - u'(1.,2.)' + '(1.,2.)' >>> _NamelistParser('5 ')._parse_literal() - u'5' + '5' >>> _NamelistParser('6.9 ')._parse_literal() - u'6.9' + '6.9' >>> _NamelistParser('inf ')._parse_literal() - u'inf' + 'inf' >>> _NamelistParser('nan(booga) ')._parse_literal() - u'nan(booga)' + 'nan(booga)' >>> _NamelistParser('.FLORIDA$ ')._parse_literal() - u'.FLORIDA$' + '.FLORIDA$' >>> _NamelistParser('hamburger ')._parse_literal() Traceback (most recent call last): ... _NamelistParseError: Error in parsing namelist: expected literal value, but got 'hamburger' >>> _NamelistParser('5,')._parse_literal() - u'5' + '5' >>> _NamelistParser('5\n')._parse_literal() - u'5' + '5' >>> _NamelistParser('5/')._parse_literal() - u'5' + '5' >>> _NamelistParser(',')._parse_literal() - u'' + '' >>> _NamelistParser('6*5 ')._parse_literal() - u'6*5' + '6*5' >>> _NamelistParser('6*(1., 2.) ')._parse_literal() - u'6*(1., 2.)' + '6*(1., 2.)' >>> _NamelistParser('6*"a" ')._parse_literal() - u'6*"a"' + '6*"a"' >>> _NamelistParser('6*')._parse_literal() Traceback (most recent call last): ... _NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser('6*')._parse_literal(allow_eof_end=True) - u'6*' + '6*' >>> _NamelistParser('foo= ')._parse_literal() Traceback (most recent call last): ... @@ -1770,15 +1770,15 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): ... _NamelistParseError: Error in parsing namelist: expected literal value, but got 'foo+=' >>> _NamelistParser('5,')._parse_literal(allow_name=True) - u'5' + '5' >>> x = _NamelistParser('foo= ') >>> x._parse_literal(allow_name=True) >>> x._curr() - u'f' + 'f' >>> x = _NamelistParser('foo+= ') >>> x._parse_literal(allow_name=True) >>> x._curr() - u'f' + 'f' >>> _NamelistParser('6*foo= ')._parse_literal(allow_name=True) Traceback (most recent call last): ... @@ -1790,13 +1790,13 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): >>> x = _NamelistParser('foo = ') >>> x._parse_literal(allow_name=True) >>> x._curr() - u'f' + 'f' >>> x = _NamelistParser('foo\n= ') >>> x._parse_literal(allow_name=True) >>> x._curr() - u'f' + 'f' >>> _NamelistParser('')._parse_literal(allow_eof_end=True) - u'' + '' """ # Deal with empty input string. if allow_eof_end and self._pos == self._len: @@ -1871,22 +1871,22 @@ def _expect_separator(self, allow_eof=False): >>> x._expect_separator() True >>> x._curr() - u'a' + 'a' >>> x = _NamelistParser(" a") >>> x._expect_separator() True >>> x._curr() - u'a' + 'a' >>> x = _NamelistParser(",a") >>> x._expect_separator() True >>> x._curr() - u'a' + 'a' >>> x = _NamelistParser("/a") >>> x._expect_separator() False >>> x._curr() - u'/' + '/' >>> x = _NamelistParser("a") >>> x._expect_separator() Traceback (most recent call last): @@ -1896,22 +1896,22 @@ def _expect_separator(self, allow_eof=False): >>> x._expect_separator() True >>> x._curr() - u'a' + 'a' >>> x = _NamelistParser(" / a") >>> x._expect_separator() False >>> x._curr() - u'/' + '/' >>> x = _NamelistParser(" , ! Some stuff\n a") >>> x._expect_separator() True >>> x._curr() - u'a' + 'a' >>> x = _NamelistParser(" , ! Some stuff\n ! Other stuff\n a") >>> x._expect_separator() True >>> x._curr() - u'a' + 'a' >>> _NamelistParser("")._expect_separator(allow_eof=True) False >>> x = _NamelistParser(" ") @@ -1966,53 +1966,53 @@ def _parse_name_and_values(self, allow_eof_end=False): alternate file format in "groupless" mode.) >>> _NamelistParser("foo='bar' /")._parse_name_and_values() - (u'foo', [u"'bar'"], False) + ('foo', ["'bar'"], False) >>> _NamelistParser("foo(3)='bar' /")._parse_name_and_values() - (u'foo(3)', [u"'bar'"], False) + ('foo(3)', ["'bar'"], False) >>> _NamelistParser("foo ='bar' /")._parse_name_and_values() - (u'foo', [u"'bar'"], False) + ('foo', ["'bar'"], False) >>> _NamelistParser("foo=\n'bar' /")._parse_name_and_values() - (u'foo', [u"'bar'"], False) + ('foo', ["'bar'"], False) >>> _NamelistParser("foo 'bar' /")._parse_name_and_values() Traceback (most recent call last): ... _NamelistParseError: Error in parsing namelist: expected '=' but found "'" >>> _NamelistParser("foo='bar','bazz' /")._parse_name_and_values() - (u'foo', [u"'bar'", u"'bazz'"], False) + ('foo', ["'bar'", "'bazz'"], False) >>> _NamelistParser("foo=,,'bazz',6*/")._parse_name_and_values() - (u'foo', [u'', u'', u"'bazz'", u'6*'], False) + ('foo', ['', '', "'bazz'", '6*'], False) >>> _NamelistParser("foo='bar' 'bazz' foo2='ban'")._parse_name_and_values() - (u'foo', [u"'bar'", u"'bazz'"], False) + ('foo', ["'bar'", "'bazz'"], False) >>> _NamelistParser("foo='bar' 'bazz' foo2(2)='ban'")._parse_name_and_values() - (u'foo', [u"'bar'", u"'bazz'"], False) + ('foo', ["'bar'", "'bazz'"], False) >>> _NamelistParser("foo= foo2='ban' ")._parse_name_and_values() Traceback (most recent call last): ... _NamelistParseError: Error in parsing namelist: expected literal value, but got "foo2='ban'" >>> _NamelistParser("foo=,,'bazz',6* ")._parse_name_and_values(allow_eof_end=True) - (u'foo', [u'', u'', u"'bazz'", u'6*'], False) + ('foo', ['', '', "'bazz'", '6*'], False) >>> _NamelistParser("foo(3)='bazz'")._parse_name_and_values(allow_eof_end=True) - (u'foo(3)', [u"'bazz'"], False) + ('foo(3)', ["'bazz'"], False) >>> _NamelistParser("foo=")._parse_name_and_values() Traceback (most recent call last): ... _NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser("foo=")._parse_name_and_values(allow_eof_end=True) - (u'foo', [u''], False) + ('foo', [''], False) >>> _NamelistParser("foo= ")._parse_name_and_values(allow_eof_end=True) - (u'foo', [u''], False) + ('foo', [''], False) >>> _NamelistParser("foo=2")._parse_name_and_values(allow_eof_end=True) - (u'foo', [u'2'], False) + ('foo', ['2'], False) >>> _NamelistParser("foo=1,2")._parse_name_and_values(allow_eof_end=True) - (u'foo', [u'1', u'2'], False) + ('foo', ['1', '2'], False) >>> _NamelistParser("foo(1:2)=1,2,3 ")._parse_name_and_values(allow_eof_end=True) Traceback (most recent call last): ... SystemExit: ERROR: Too many values for array foo(1:2) >>> _NamelistParser("foo=1,")._parse_name_and_values(allow_eof_end=True) - (u'foo', [u'1', u''], False) + ('foo', ['1', ''], False) >>> _NamelistParser("foo+=1")._parse_name_and_values(allow_eof_end=True) - (u'foo', [u'1'], True) + ('foo', ['1'], True) """ name = self._parse_variable_name() addto = False # This keeps track of whether += existed @@ -2059,19 +2059,19 @@ def _parse_namelist_group(self): >>> x = _NamelistParser("&group /") >>> x._parse_namelist_group() >>> x._settings - OrderedDict([(u'group', {})]) + OrderedDict([('group', {})]) >>> x._curr() - u'/' + '/' >>> x = _NamelistParser("&group\n foo='bar','bazz'\n,, foo2=2*5\n /") >>> x._parse_namelist_group() >>> x._settings - OrderedDict([(u'group', {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'5', u'5']})]) + OrderedDict([('group', {'foo': ["'bar'", "'bazz'", ''], 'foo2': ['5', '5']})]) >>> x = _NamelistParser("&group\n foo='bar','bazz'\n,, foo2=2*5\n /", groupless=True) >>> x._parse_namelist_group() >>> x._settings - OrderedDict([(u'foo', [u"'bar'", u"'bazz'", u'']), (u'foo2', [u'5', u'5'])]) + OrderedDict([('foo', ["'bar'", "'bazz'", '']), ('foo2', ['5', '5'])]) >>> x._curr() - u'/' + '/' >>> x = _NamelistParser("&group /&group /") >>> x._parse_namelist_group() >>> x._advance() @@ -2082,31 +2082,31 @@ def _parse_namelist_group(self): >>> x = _NamelistParser("&group foo='bar', foo='bazz' /") >>> x._parse_namelist_group() >>> x._settings - OrderedDict([(u'group', {u'foo': [u"'bazz'"]})]) + OrderedDict([('group', {'foo': ["'bazz'"]})]) >>> x = _NamelistParser("&group foo='bar', foo= /") >>> x._parse_namelist_group() >>> x._settings - OrderedDict([(u'group', {u'foo': [u"'bar'"]})]) + OrderedDict([('group', {'foo': ["'bar'"]})]) >>> x = _NamelistParser("&group foo='bar', foo= /", groupless=True) >>> x._parse_namelist_group() >>> x._settings - OrderedDict([(u'foo', [u"'bar'"])]) + OrderedDict([('foo', ["'bar'"])]) >>> x = _NamelistParser("&group foo='bar', foo+='baz' /", groupless=True) >>> x._parse_namelist_group() >>> x._settings - OrderedDict([(u'foo', [u"'bar'", u"'baz'"])]) + OrderedDict([('foo', ["'bar'", "'baz'"])]) >>> x = _NamelistParser("&group foo+='bar' /", groupless=True) >>> x._parse_namelist_group() >>> x._settings - OrderedDict([(u'foo', [u"'bar'"])]) + OrderedDict([('foo', ["'bar'"])]) >>> x = _NamelistParser("&group foo='bar', foo+='baz' /") >>> x._parse_namelist_group() >>> x._settings - OrderedDict([(u'group', {u'foo': [u"'bar'", u"'baz'"]})]) + OrderedDict([('group', {'foo': ["'bar'", "'baz'"]})]) >>> x = _NamelistParser("&group foo+='bar' /") >>> x._parse_namelist_group() >>> x._settings - OrderedDict([(u'group', {u'foo': [u"'bar'"]})]) + OrderedDict([('group', {'foo': ["'bar'"]})]) """ group_name = self._parse_namelist_group_name() if not self._groupless: @@ -2147,37 +2147,37 @@ def parse_namelist(self): >>> _NamelistParser(" \n!Comment").parse_namelist() OrderedDict() >>> _NamelistParser(" &group /").parse_namelist() - OrderedDict([(u'group', {})]) + OrderedDict([('group', {})]) >>> _NamelistParser("! Comment \n &group /! Comment\n ").parse_namelist() - OrderedDict([(u'group', {})]) + OrderedDict([('group', {})]) >>> _NamelistParser("! Comment \n &group /! Comment ").parse_namelist() - OrderedDict([(u'group', {})]) + OrderedDict([('group', {})]) >>> _NamelistParser("&group1\n foo='bar','bazz'\n,, foo2=2*5\n / &group2 /").parse_namelist() - OrderedDict([(u'group1', {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'5', u'5']}), (u'group2', {})]) + OrderedDict([('group1', {'foo': ["'bar'", "'bazz'", ''], 'foo2': ['5', '5']}), ('group2', {})]) >>> _NamelistParser("!blah \n foo='bar','bazz'\n,, foo2=2*5\n ", groupless=True).parse_namelist() - OrderedDict([(u'foo', [u"'bar'", u"'bazz'", u'']), (u'foo2', [u'2*5'])]) + OrderedDict([('foo', ["'bar'", "'bazz'", '']), ('foo2', ['2*5'])]) >>> _NamelistParser("!blah \n foo='bar','bazz'\n,, foo2=2*5,6\n ", groupless=True).parse_namelist() - OrderedDict([(u'foo', [u"'bar'", u"'bazz'", u'']), (u'foo2', [u'2*5', u'6'])]) + OrderedDict([('foo', ["'bar'", "'bazz'", '']), ('foo2', ['2*5', '6'])]) >>> _NamelistParser("!blah \n foo='bar'", groupless=True).parse_namelist() - OrderedDict([(u'foo', [u"'bar'"])]) + OrderedDict([('foo', ["'bar'"])]) >>> _NamelistParser("foo='bar', foo(3)='bazz'", groupless=True).parse_namelist() - OrderedDict([(u'foo', [u"'bar'"]), (u'foo(3)', [u"'bazz'"])]) + OrderedDict([('foo', ["'bar'"]), ('foo(3)', ["'bazz'"])]) >>> _NamelistParser("foo(2)='bar'", groupless=True).parse_namelist() - OrderedDict([(u'foo(2)', [u"'bar'"])]) + OrderedDict([('foo(2)', ["'bar'"])]) >>> _NamelistParser("foo(2)='bar', foo(3)='bazz'", groupless=True).parse_namelist() - OrderedDict([(u'foo(2)', [u"'bar'"]), (u'foo(3)', [u"'bazz'"])]) + OrderedDict([('foo(2)', ["'bar'"]), ('foo(3)', ["'bazz'"])]) >>> _NamelistParser("foo='bar', foo='bazz'", groupless=True).parse_namelist() - OrderedDict([(u'foo', [u"'bazz'"])]) + OrderedDict([('foo', ["'bazz'"])]) >>> _NamelistParser("foo='bar'\n foo+='bazz'", groupless=True).parse_namelist() - OrderedDict([(u'foo', [u"'bar'", u"'bazz'"])]) + OrderedDict([('foo', ["'bar'", "'bazz'"])]) >>> _NamelistParser("foo='bar', foo='bazz'", groupless=True).parse_namelist() - OrderedDict([(u'foo', [u"'bazz'"])]) + OrderedDict([('foo', ["'bazz'"])]) >>> _NamelistParser("foo='bar', foo=", groupless=True).parse_namelist() - OrderedDict([(u'foo', [u"'bar'"])]) + OrderedDict([('foo', ["'bar'"])]) >>> _NamelistParser("foo='bar', 'bazz'\n foo+='ban'", groupless=True).parse_namelist() - OrderedDict([(u'foo', [u"'bar'", u"'bazz'", u"'ban'"])]) + OrderedDict([('foo', ["'bar'", "'bazz'", "'ban'"])]) >>> _NamelistParser("foo+='bar'", groupless=True).parse_namelist() - OrderedDict([(u'foo', [u"'bar'"])]) + OrderedDict([('foo', ["'bar'"])]) """ # Return empty dictionary for empty files. if self._len == 0: diff --git a/scripts/lib/CIME/utils.py b/scripts/lib/CIME/utils.py index 76b384a94b8..4b09cd197ce 100644 --- a/scripts/lib/CIME/utils.py +++ b/scripts/lib/CIME/utils.py @@ -155,7 +155,7 @@ def get_model(): >>> del os.environ["CIME_MODEL"] >>> set_model('rocky') >>> get_model() - 'rocky' + u'rocky' >>> reset_cime_config() """ model = os.environ.get("CIME_MODEL") diff --git a/scripts/lib/CIME/wait_for_tests.py b/scripts/lib/CIME/wait_for_tests.py index 85f6265db5f..134efb69475 100644 --- a/scripts/lib/CIME/wait_for_tests.py +++ b/scripts/lib/CIME/wait_for_tests.py @@ -1,4 +1,5 @@ -import os, time, threading, queue, socket, signal, distutils.spawn, shutil, glob +from CIME.six.moves import queue +import os, time, threading, socket, signal, distutils.spawn, shutil, glob import logging import xml.etree.ElementTree as xmlet @@ -7,6 +8,7 @@ from CIME.XML.machines import Machines from CIME.test_status import * + SIGNAL_RECEIVED = False ACME_MAIN_CDASH = "ACME_Climate" CDASH_DEFAULT_BUILD_GROUP = "ACME_Latest" diff --git a/scripts/tests/scripts_regression_tests.py b/scripts/tests/scripts_regression_tests.py index 7ef9bb14a74..0be3bcd5e87 100755 --- a/scripts/tests/scripts_regression_tests.py +++ b/scripts/tests/scripts_regression_tests.py @@ -1692,7 +1692,7 @@ def get_macros(macro_maker, build_xml, build_system): """ # Build.write_macros expects file-like objects as input, so # we need to wrap the strings in StringIO objects. - xml = io.StringIO(str(build_xml)) + xml = io.StringIO(unicode(build_xml)) output = io.StringIO() output_format = None if build_system == "Makefile": From 67c62694ab8cb1e62e1a68d8ab8644f93785f908 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 3 Oct 2017 17:14:04 -0600 Subject: [PATCH 10/41] passes B_CheckCode --- scripts/Tools/cimeteststatus | 15 +++++++-------- scripts/Tools/code_checker | 2 +- scripts/Tools/cs.status | 5 +++-- scripts/lib/CIME/XML/test_reporter.py | 3 ++- scripts/lib/CIME/wait_for_tests.py | 1 + 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/scripts/Tools/cimeteststatus b/scripts/Tools/cimeteststatus index 7795a01e255..1c1daf7ffa4 100755 --- a/scripts/Tools/cimeteststatus +++ b/scripts/Tools/cimeteststatus @@ -5,9 +5,9 @@ Purpose: Give basic and detailed summaries of CIME(CESM) tests, and send the test results back to the test database. Authors: Jay Shollenberger and Ben Andre """ - - -import sys +from standard_script_setup import * +#pylint: disable=import-error +from CIME.six.moves import urllib if sys.hexversion < 0x02070000: print(70 * "*") print("ERROR: {0} requires python >= 2.7.x. ".format(sys.argv[0])) @@ -17,12 +17,11 @@ if sys.hexversion < 0x02070000: sys.exit(1) import xml.etree.ElementTree as etree import argparse -import os, glob, re -import urllib.request, urllib.parse, urllib.error -import urllib.request, urllib.error, urllib.parse +import glob, re import pprint import getpass + testdburl = "https://csegweb.cgd.ucar.edu/testdb/cgi-bin/processXMLtest.cgi" class CimeTestStatus(): @@ -552,7 +551,7 @@ def authenticate(): auth['password'] = getpass.getpass("enter TestDB password:") return auth -def main(): +def _main_func(): """ Parse the arguments, get the suite information from the test spec, get the test statuses, then print a raw status, test summary, or send the test report. @@ -596,5 +595,5 @@ def main(): if __name__ == "__main__": - main() + _main_func() diff --git a/scripts/Tools/code_checker b/scripts/Tools/code_checker index f57dfa2ab45..f5d63db1247 100755 --- a/scripts/Tools/code_checker +++ b/scripts/Tools/code_checker @@ -5,7 +5,7 @@ Ensure that all CIME python files are free of errors and follow the PEP8 standard. """ -from .standard_script_setup import * +from standard_script_setup import * from CIME.code_checker import check_code, expect diff --git a/scripts/Tools/cs.status b/scripts/Tools/cs.status index 38799631193..2c57dc7652e 100755 --- a/scripts/Tools/cs.status +++ b/scripts/Tools/cs.status @@ -4,7 +4,7 @@ List test results based on TestStatus files. Returns True if no errors occured (not based on test statuses). """ -from .standard_script_setup import * +from standard_script_setup import * import argparse, sys, os, logging, glob from CIME.test_status import * @@ -66,7 +66,8 @@ def cs_status(test_paths, summary=False): for test_id in sorted(test_id_output): print(test_id) - print(test_id_output[test_id], end=' ') + print(test_id_output[test_id]) + print(' ') ############################################################################### def _main_func(description): diff --git a/scripts/lib/CIME/XML/test_reporter.py b/scripts/lib/CIME/XML/test_reporter.py index c0a5987c186..c711bc63827 100644 --- a/scripts/lib/CIME/XML/test_reporter.py +++ b/scripts/lib/CIME/XML/test_reporter.py @@ -2,7 +2,8 @@ Interface to the testreporter xml. This class inherits from GenericXML.py """ -from CIME.six.moves.urllib import request, parse, error +#pylint: disable=import-error +from CIME.six.moves import urllib from CIME.XML.standard_module_setup import * from CIME.XML.generic_xml import GenericXML from CIME.utils import expect,get_model diff --git a/scripts/lib/CIME/wait_for_tests.py b/scripts/lib/CIME/wait_for_tests.py index 134efb69475..9f2a746a4af 100644 --- a/scripts/lib/CIME/wait_for_tests.py +++ b/scripts/lib/CIME/wait_for_tests.py @@ -1,3 +1,4 @@ +#pylint: disable=import-error from CIME.six.moves import queue import os, time, threading, socket, signal, distutils.spawn, shutil, glob import logging From c536f4f14fb49b11302f4fc8be5d84b64104a3ca Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 3 Oct 2017 17:43:25 -0600 Subject: [PATCH 11/41] create_newcase works --- scripts/lib/CIME/case.py | 2 +- scripts/lib/CIME/utils.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index 67fdd40cdd7..f3162c1865b 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -7,7 +7,7 @@ from copy import deepcopy import glob, os, shutil, math from CIME.XML.standard_module_setup import * - +from CIME.six.moves import input from CIME.utils import expect, get_cime_root, append_status from CIME.utils import convert_to_type, get_model, get_project from CIME.utils import get_current_commit, check_name diff --git a/scripts/lib/CIME/utils.py b/scripts/lib/CIME/utils.py index 4b09cd197ce..5dc1d2f4180 100644 --- a/scripts/lib/CIME/utils.py +++ b/scripts/lib/CIME/utils.py @@ -5,6 +5,7 @@ import logging, gzip, sys, os, time, re, shutil, glob, string, random, imp, errno, signal import stat as statlib import warnings +from CIME import six from contextlib import contextmanager # Return this error code if the scripts worked but tests failed @@ -856,9 +857,9 @@ def convert_to_string(value, type_str=None, vid=""): """ if value is not None and type(value) is not str: if type_str == "char": - expect(type(value) is str, "Wrong type for entry id '{}'".format(vid)) + expect(isinstance(value, six.string_types), "Wrong type for entry id '{}'".format(vid)) elif type_str == "integer": - expect(type(value) is int, "Wrong type for entry id '{}'".format(vid)) + expect(isinstance(value, six.integer_types), "Wrong type for entry id '{}'".format(vid)) value = str(value) elif type_str == "logical": expect(type(value) is bool, "Wrong type for entry id '{}'".format(vid)) From 2437182c62bd65f1f3d879d4f2cdeca9c8b81fd0 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 3 Oct 2017 17:44:04 -0600 Subject: [PATCH 12/41] add the 2x3 module six.py --- scripts/lib/CIME/six.py | 891 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 891 insertions(+) create mode 100644 scripts/lib/CIME/six.py diff --git a/scripts/lib/CIME/six.py b/scripts/lib/CIME/six.py new file mode 100644 index 00000000000..6bf4fd38104 --- /dev/null +++ b/scripts/lib/CIME/six.py @@ -0,0 +1,891 @@ +# Copyright (c) 2010-2017 Benjamin Peterson +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +"""Utilities for writing code that runs on Python 2 and 3""" + +from __future__ import absolute_import + +import functools +import itertools +import operator +import sys +import types + +__author__ = "Benjamin Peterson " +__version__ = "1.11.0" + + +# Useful for very coarse version differentiation. +PY2 = sys.version_info[0] == 2 +PY3 = sys.version_info[0] == 3 +PY34 = sys.version_info[0:2] >= (3, 4) + +if PY3: + string_types = str, + integer_types = int, + class_types = type, + text_type = str + binary_type = bytes + + MAXSIZE = sys.maxsize +else: + string_types = basestring, + integer_types = (int, long) + class_types = (type, types.ClassType) + text_type = unicode + binary_type = str + + if sys.platform.startswith("java"): + # Jython always uses 32 bits. + MAXSIZE = int((1 << 31) - 1) + else: + # It's possible to have sizeof(long) != sizeof(Py_ssize_t). + class X(object): + + def __len__(self): + return 1 << 31 + try: + len(X()) + except OverflowError: + # 32-bit + MAXSIZE = int((1 << 31) - 1) + else: + # 64-bit + MAXSIZE = int((1 << 63) - 1) + del X + + +def _add_doc(func, doc): + """Add documentation to a function.""" + func.__doc__ = doc + + +def _import_module(name): + """Import module, returning the module after the last dot.""" + __import__(name) + return sys.modules[name] + + +class _LazyDescr(object): + + def __init__(self, name): + self.name = name + + def __get__(self, obj, tp): + result = self._resolve() + setattr(obj, self.name, result) # Invokes __set__. + try: + # This is a bit ugly, but it avoids running this again by + # removing this descriptor. + delattr(obj.__class__, self.name) + except AttributeError: + pass + return result + + +class MovedModule(_LazyDescr): + + def __init__(self, name, old, new=None): + super(MovedModule, self).__init__(name) + if PY3: + if new is None: + new = name + self.mod = new + else: + self.mod = old + + def _resolve(self): + return _import_module(self.mod) + + def __getattr__(self, attr): + _module = self._resolve() + value = getattr(_module, attr) + setattr(self, attr, value) + return value + + +class _LazyModule(types.ModuleType): + + def __init__(self, name): + super(_LazyModule, self).__init__(name) + self.__doc__ = self.__class__.__doc__ + + def __dir__(self): + attrs = ["__doc__", "__name__"] + attrs += [attr.name for attr in self._moved_attributes] + return attrs + + # Subclasses should override this + _moved_attributes = [] + + +class MovedAttribute(_LazyDescr): + + def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None): + super(MovedAttribute, self).__init__(name) + if PY3: + if new_mod is None: + new_mod = name + self.mod = new_mod + if new_attr is None: + if old_attr is None: + new_attr = name + else: + new_attr = old_attr + self.attr = new_attr + else: + self.mod = old_mod + if old_attr is None: + old_attr = name + self.attr = old_attr + + def _resolve(self): + module = _import_module(self.mod) + return getattr(module, self.attr) + + +class _SixMetaPathImporter(object): + + """ + A meta path importer to import six.moves and its submodules. + + This class implements a PEP302 finder and loader. It should be compatible + with Python 2.5 and all existing versions of Python3 + """ + + def __init__(self, six_module_name): + self.name = six_module_name + self.known_modules = {} + + def _add_module(self, mod, *fullnames): + for fullname in fullnames: + self.known_modules[self.name + "." + fullname] = mod + + def _get_module(self, fullname): + return self.known_modules[self.name + "." + fullname] + + def find_module(self, fullname, path=None): + if fullname in self.known_modules: + return self + return None + + def __get_module(self, fullname): + try: + return self.known_modules[fullname] + except KeyError: + raise ImportError("This loader does not know module " + fullname) + + def load_module(self, fullname): + try: + # in case of a reload + return sys.modules[fullname] + except KeyError: + pass + mod = self.__get_module(fullname) + if isinstance(mod, MovedModule): + mod = mod._resolve() + else: + mod.__loader__ = self + sys.modules[fullname] = mod + return mod + + def is_package(self, fullname): + """ + Return true, if the named module is a package. + + We need this method to get correct spec objects with + Python 3.4 (see PEP451) + """ + return hasattr(self.__get_module(fullname), "__path__") + + def get_code(self, fullname): + """Return None + + Required, if is_package is implemented""" + self.__get_module(fullname) # eventually raises ImportError + return None + get_source = get_code # same as get_code + +_importer = _SixMetaPathImporter(__name__) + + +class _MovedItems(_LazyModule): + + """Lazy loading of moved objects""" + __path__ = [] # mark as package + + +_moved_attributes = [ + MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), + MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), + MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"), + MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), + MovedAttribute("intern", "__builtin__", "sys"), + MovedAttribute("map", "itertools", "builtins", "imap", "map"), + MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"), + MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"), + MovedAttribute("getoutput", "commands", "subprocess"), + MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), + MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"), + MovedAttribute("reduce", "__builtin__", "functools"), + MovedAttribute("shlex_quote", "pipes", "shlex", "quote"), + MovedAttribute("StringIO", "StringIO", "io"), + MovedAttribute("UserDict", "UserDict", "collections"), + MovedAttribute("UserList", "UserList", "collections"), + MovedAttribute("UserString", "UserString", "collections"), + MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), + MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), + MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), + MovedModule("builtins", "__builtin__"), + MovedModule("configparser", "ConfigParser"), + MovedModule("copyreg", "copy_reg"), + MovedModule("dbm_gnu", "gdbm", "dbm.gnu"), + MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"), + MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), + MovedModule("http_cookies", "Cookie", "http.cookies"), + MovedModule("html_entities", "htmlentitydefs", "html.entities"), + MovedModule("html_parser", "HTMLParser", "html.parser"), + MovedModule("http_client", "httplib", "http.client"), + MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), + MovedModule("email_mime_image", "email.MIMEImage", "email.mime.image"), + MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), + MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"), + MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), + MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), + MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), + MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), + MovedModule("cPickle", "cPickle", "pickle"), + MovedModule("queue", "Queue"), + MovedModule("reprlib", "repr"), + MovedModule("socketserver", "SocketServer"), + MovedModule("_thread", "thread", "_thread"), + MovedModule("tkinter", "Tkinter"), + MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"), + MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"), + MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"), + MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"), + MovedModule("tkinter_tix", "Tix", "tkinter.tix"), + MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"), + MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"), + MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"), + MovedModule("tkinter_colorchooser", "tkColorChooser", + "tkinter.colorchooser"), + MovedModule("tkinter_commondialog", "tkCommonDialog", + "tkinter.commondialog"), + MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"), + MovedModule("tkinter_font", "tkFont", "tkinter.font"), + MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"), + MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", + "tkinter.simpledialog"), + MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"), + MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"), + MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"), + MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"), + MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"), + MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"), +] +# Add windows specific modules. +if sys.platform == "win32": + _moved_attributes += [ + MovedModule("winreg", "_winreg"), + ] + +for attr in _moved_attributes: + setattr(_MovedItems, attr.name, attr) + if isinstance(attr, MovedModule): + _importer._add_module(attr, "moves." + attr.name) +del attr + +_MovedItems._moved_attributes = _moved_attributes + +moves = _MovedItems(__name__ + ".moves") +_importer._add_module(moves, "moves") + + +class Module_six_moves_urllib_parse(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_parse""" + + +_urllib_parse_moved_attributes = [ + MovedAttribute("ParseResult", "urlparse", "urllib.parse"), + MovedAttribute("SplitResult", "urlparse", "urllib.parse"), + MovedAttribute("parse_qs", "urlparse", "urllib.parse"), + MovedAttribute("parse_qsl", "urlparse", "urllib.parse"), + MovedAttribute("urldefrag", "urlparse", "urllib.parse"), + MovedAttribute("urljoin", "urlparse", "urllib.parse"), + MovedAttribute("urlparse", "urlparse", "urllib.parse"), + MovedAttribute("urlsplit", "urlparse", "urllib.parse"), + MovedAttribute("urlunparse", "urlparse", "urllib.parse"), + MovedAttribute("urlunsplit", "urlparse", "urllib.parse"), + MovedAttribute("quote", "urllib", "urllib.parse"), + MovedAttribute("quote_plus", "urllib", "urllib.parse"), + MovedAttribute("unquote", "urllib", "urllib.parse"), + MovedAttribute("unquote_plus", "urllib", "urllib.parse"), + MovedAttribute("unquote_to_bytes", "urllib", "urllib.parse", "unquote", "unquote_to_bytes"), + MovedAttribute("urlencode", "urllib", "urllib.parse"), + MovedAttribute("splitquery", "urllib", "urllib.parse"), + MovedAttribute("splittag", "urllib", "urllib.parse"), + MovedAttribute("splituser", "urllib", "urllib.parse"), + MovedAttribute("splitvalue", "urllib", "urllib.parse"), + MovedAttribute("uses_fragment", "urlparse", "urllib.parse"), + MovedAttribute("uses_netloc", "urlparse", "urllib.parse"), + MovedAttribute("uses_params", "urlparse", "urllib.parse"), + MovedAttribute("uses_query", "urlparse", "urllib.parse"), + MovedAttribute("uses_relative", "urlparse", "urllib.parse"), +] +for attr in _urllib_parse_moved_attributes: + setattr(Module_six_moves_urllib_parse, attr.name, attr) +del attr + +Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes + +_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"), + "moves.urllib_parse", "moves.urllib.parse") + + +class Module_six_moves_urllib_error(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_error""" + + +_urllib_error_moved_attributes = [ + MovedAttribute("URLError", "urllib2", "urllib.error"), + MovedAttribute("HTTPError", "urllib2", "urllib.error"), + MovedAttribute("ContentTooShortError", "urllib", "urllib.error"), +] +for attr in _urllib_error_moved_attributes: + setattr(Module_six_moves_urllib_error, attr.name, attr) +del attr + +Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes + +_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"), + "moves.urllib_error", "moves.urllib.error") + + +class Module_six_moves_urllib_request(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_request""" + + +_urllib_request_moved_attributes = [ + MovedAttribute("urlopen", "urllib2", "urllib.request"), + MovedAttribute("install_opener", "urllib2", "urllib.request"), + MovedAttribute("build_opener", "urllib2", "urllib.request"), + MovedAttribute("pathname2url", "urllib", "urllib.request"), + MovedAttribute("url2pathname", "urllib", "urllib.request"), + MovedAttribute("getproxies", "urllib", "urllib.request"), + MovedAttribute("Request", "urllib2", "urllib.request"), + MovedAttribute("OpenerDirector", "urllib2", "urllib.request"), + MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"), + MovedAttribute("ProxyHandler", "urllib2", "urllib.request"), + MovedAttribute("BaseHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"), + MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"), + MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"), + MovedAttribute("FileHandler", "urllib2", "urllib.request"), + MovedAttribute("FTPHandler", "urllib2", "urllib.request"), + MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"), + MovedAttribute("UnknownHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"), + MovedAttribute("urlretrieve", "urllib", "urllib.request"), + MovedAttribute("urlcleanup", "urllib", "urllib.request"), + MovedAttribute("URLopener", "urllib", "urllib.request"), + MovedAttribute("FancyURLopener", "urllib", "urllib.request"), + MovedAttribute("proxy_bypass", "urllib", "urllib.request"), + MovedAttribute("parse_http_list", "urllib2", "urllib.request"), + MovedAttribute("parse_keqv_list", "urllib2", "urllib.request"), +] +for attr in _urllib_request_moved_attributes: + setattr(Module_six_moves_urllib_request, attr.name, attr) +del attr + +Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes + +_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"), + "moves.urllib_request", "moves.urllib.request") + + +class Module_six_moves_urllib_response(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_response""" + + +_urllib_response_moved_attributes = [ + MovedAttribute("addbase", "urllib", "urllib.response"), + MovedAttribute("addclosehook", "urllib", "urllib.response"), + MovedAttribute("addinfo", "urllib", "urllib.response"), + MovedAttribute("addinfourl", "urllib", "urllib.response"), +] +for attr in _urllib_response_moved_attributes: + setattr(Module_six_moves_urllib_response, attr.name, attr) +del attr + +Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes + +_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"), + "moves.urllib_response", "moves.urllib.response") + + +class Module_six_moves_urllib_robotparser(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_robotparser""" + + +_urllib_robotparser_moved_attributes = [ + MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"), +] +for attr in _urllib_robotparser_moved_attributes: + setattr(Module_six_moves_urllib_robotparser, attr.name, attr) +del attr + +Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes + +_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"), + "moves.urllib_robotparser", "moves.urllib.robotparser") + + +class Module_six_moves_urllib(types.ModuleType): + + """Create a six.moves.urllib namespace that resembles the Python 3 namespace""" + __path__ = [] # mark as package + parse = _importer._get_module("moves.urllib_parse") + error = _importer._get_module("moves.urllib_error") + request = _importer._get_module("moves.urllib_request") + response = _importer._get_module("moves.urllib_response") + robotparser = _importer._get_module("moves.urllib_robotparser") + + def __dir__(self): + return ['parse', 'error', 'request', 'response', 'robotparser'] + +_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"), + "moves.urllib") + + +def add_move(move): + """Add an item to six.moves.""" + setattr(_MovedItems, move.name, move) + + +def remove_move(name): + """Remove item from six.moves.""" + try: + delattr(_MovedItems, name) + except AttributeError: + try: + del moves.__dict__[name] + except KeyError: + raise AttributeError("no such move, %r" % (name,)) + + +if PY3: + _meth_func = "__func__" + _meth_self = "__self__" + + _func_closure = "__closure__" + _func_code = "__code__" + _func_defaults = "__defaults__" + _func_globals = "__globals__" +else: + _meth_func = "im_func" + _meth_self = "im_self" + + _func_closure = "func_closure" + _func_code = "func_code" + _func_defaults = "func_defaults" + _func_globals = "func_globals" + + +try: + advance_iterator = next +except NameError: + def advance_iterator(it): + return it.next() +next = advance_iterator + + +try: + callable = callable +except NameError: + def callable(obj): + return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) + + +if PY3: + def get_unbound_function(unbound): + return unbound + + create_bound_method = types.MethodType + + def create_unbound_method(func, cls): + return func + + Iterator = object +else: + def get_unbound_function(unbound): + return unbound.im_func + + def create_bound_method(func, obj): + return types.MethodType(func, obj, obj.__class__) + + def create_unbound_method(func, cls): + return types.MethodType(func, None, cls) + + class Iterator(object): + + def next(self): + return type(self).__next__(self) + + callable = callable +_add_doc(get_unbound_function, + """Get the function out of a possibly unbound function""") + + +get_method_function = operator.attrgetter(_meth_func) +get_method_self = operator.attrgetter(_meth_self) +get_function_closure = operator.attrgetter(_func_closure) +get_function_code = operator.attrgetter(_func_code) +get_function_defaults = operator.attrgetter(_func_defaults) +get_function_globals = operator.attrgetter(_func_globals) + + +if PY3: + def iterkeys(d, **kw): + return iter(d.keys(**kw)) + + def itervalues(d, **kw): + return iter(d.values(**kw)) + + def iteritems(d, **kw): + return iter(d.items(**kw)) + + def iterlists(d, **kw): + return iter(d.lists(**kw)) + + viewkeys = operator.methodcaller("keys") + + viewvalues = operator.methodcaller("values") + + viewitems = operator.methodcaller("items") +else: + def iterkeys(d, **kw): + return d.iterkeys(**kw) + + def itervalues(d, **kw): + return d.itervalues(**kw) + + def iteritems(d, **kw): + return d.iteritems(**kw) + + def iterlists(d, **kw): + return d.iterlists(**kw) + + viewkeys = operator.methodcaller("viewkeys") + + viewvalues = operator.methodcaller("viewvalues") + + viewitems = operator.methodcaller("viewitems") + +_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.") +_add_doc(itervalues, "Return an iterator over the values of a dictionary.") +_add_doc(iteritems, + "Return an iterator over the (key, value) pairs of a dictionary.") +_add_doc(iterlists, + "Return an iterator over the (key, [values]) pairs of a dictionary.") + + +if PY3: + def b(s): + return s.encode("latin-1") + + def u(s): + return s + unichr = chr + import struct + int2byte = struct.Struct(">B").pack + del struct + byte2int = operator.itemgetter(0) + indexbytes = operator.getitem + iterbytes = iter + import io + StringIO = io.StringIO + BytesIO = io.BytesIO + _assertCountEqual = "assertCountEqual" + if sys.version_info[1] <= 1: + _assertRaisesRegex = "assertRaisesRegexp" + _assertRegex = "assertRegexpMatches" + else: + _assertRaisesRegex = "assertRaisesRegex" + _assertRegex = "assertRegex" +else: + def b(s): + return s + # Workaround for standalone backslash + + def u(s): + return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape") + unichr = unichr + int2byte = chr + + def byte2int(bs): + return ord(bs[0]) + + def indexbytes(buf, i): + return ord(buf[i]) + iterbytes = functools.partial(itertools.imap, ord) + import StringIO + StringIO = BytesIO = StringIO.StringIO + _assertCountEqual = "assertItemsEqual" + _assertRaisesRegex = "assertRaisesRegexp" + _assertRegex = "assertRegexpMatches" +_add_doc(b, """Byte literal""") +_add_doc(u, """Text literal""") + + +def assertCountEqual(self, *args, **kwargs): + return getattr(self, _assertCountEqual)(*args, **kwargs) + + +def assertRaisesRegex(self, *args, **kwargs): + return getattr(self, _assertRaisesRegex)(*args, **kwargs) + + +def assertRegex(self, *args, **kwargs): + return getattr(self, _assertRegex)(*args, **kwargs) + + +if PY3: + exec_ = getattr(moves.builtins, "exec") + + def reraise(tp, value, tb=None): + try: + if value is None: + value = tp() + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + finally: + value = None + tb = None + +else: + def exec_(_code_, _globs_=None, _locs_=None): + """Execute code in a namespace.""" + if _globs_ is None: + frame = sys._getframe(1) + _globs_ = frame.f_globals + if _locs_ is None: + _locs_ = frame.f_locals + del frame + elif _locs_ is None: + _locs_ = _globs_ + exec("""exec _code_ in _globs_, _locs_""") + + exec_("""def reraise(tp, value, tb=None): + try: + raise tp, value, tb + finally: + tb = None +""") + + +if sys.version_info[:2] == (3, 2): + exec_("""def raise_from(value, from_value): + try: + if from_value is None: + raise value + raise value from from_value + finally: + value = None +""") +elif sys.version_info[:2] > (3, 2): + exec_("""def raise_from(value, from_value): + try: + raise value from from_value + finally: + value = None +""") +else: + def raise_from(value, from_value): + raise value + + +print_ = getattr(moves.builtins, "print", None) +if print_ is None: + def print_(*args, **kwargs): + """The new-style print function for Python 2.4 and 2.5.""" + fp = kwargs.pop("file", sys.stdout) + if fp is None: + return + + def write(data): + if not isinstance(data, basestring): + data = str(data) + # If the file has an encoding, encode unicode with it. + if (isinstance(fp, file) and + isinstance(data, unicode) and + fp.encoding is not None): + errors = getattr(fp, "errors", None) + if errors is None: + errors = "strict" + data = data.encode(fp.encoding, errors) + fp.write(data) + want_unicode = False + sep = kwargs.pop("sep", None) + if sep is not None: + if isinstance(sep, unicode): + want_unicode = True + elif not isinstance(sep, str): + raise TypeError("sep must be None or a string") + end = kwargs.pop("end", None) + if end is not None: + if isinstance(end, unicode): + want_unicode = True + elif not isinstance(end, str): + raise TypeError("end must be None or a string") + if kwargs: + raise TypeError("invalid keyword arguments to print()") + if not want_unicode: + for arg in args: + if isinstance(arg, unicode): + want_unicode = True + break + if want_unicode: + newline = unicode("\n") + space = unicode(" ") + else: + newline = "\n" + space = " " + if sep is None: + sep = space + if end is None: + end = newline + for i, arg in enumerate(args): + if i: + write(sep) + write(arg) + write(end) +if sys.version_info[:2] < (3, 3): + _print = print_ + + def print_(*args, **kwargs): + fp = kwargs.get("file", sys.stdout) + flush = kwargs.pop("flush", False) + _print(*args, **kwargs) + if flush and fp is not None: + fp.flush() + +_add_doc(reraise, """Reraise an exception.""") + +if sys.version_info[0:2] < (3, 4): + def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS, + updated=functools.WRAPPER_UPDATES): + def wrapper(f): + f = functools.wraps(wrapped, assigned, updated)(f) + f.__wrapped__ = wrapped + return f + return wrapper +else: + wraps = functools.wraps + + +def with_metaclass(meta, *bases): + """Create a base class with a metaclass.""" + # This requires a bit of explanation: the basic idea is to make a dummy + # metaclass for one level of class instantiation that replaces itself with + # the actual metaclass. + class metaclass(type): + + def __new__(cls, name, this_bases, d): + return meta(name, bases, d) + + @classmethod + def __prepare__(cls, name, this_bases): + return meta.__prepare__(name, bases) + return type.__new__(metaclass, 'temporary_class', (), {}) + + +def add_metaclass(metaclass): + """Class decorator for creating a class with a metaclass.""" + def wrapper(cls): + orig_vars = cls.__dict__.copy() + slots = orig_vars.get('__slots__') + if slots is not None: + if isinstance(slots, str): + slots = [slots] + for slots_var in slots: + orig_vars.pop(slots_var) + orig_vars.pop('__dict__', None) + orig_vars.pop('__weakref__', None) + return metaclass(cls.__name__, cls.__bases__, orig_vars) + return wrapper + + +def python_2_unicode_compatible(klass): + """ + A decorator that defines __unicode__ and __str__ methods under Python 2. + Under Python 3 it does nothing. + + To support Python 2 and 3 with a single code base, define a __str__ method + returning text and apply this decorator to the class. + """ + if PY2: + if '__str__' not in klass.__dict__: + raise ValueError("@python_2_unicode_compatible cannot be applied " + "to %s because it doesn't define __str__()." % + klass.__name__) + klass.__unicode__ = klass.__str__ + klass.__str__ = lambda self: self.__unicode__().encode('utf-8') + return klass + + +# Complete the moves implementation. +# This code is at the end of this module to speed up module loading. +# Turn this module into a package. +__path__ = [] # required for PEP 302 and PEP 451 +__package__ = __name__ # see PEP 366 @ReservedAssignment +if globals().get("__spec__") is not None: + __spec__.submodule_search_locations = [] # PEP 451 @UndefinedVariable +# Remove other six meta path importers, since they cause problems. This can +# happen if six is removed from sys.modules and then reloaded. (Setuptools does +# this for some reason.) +if sys.meta_path: + for i, importer in enumerate(sys.meta_path): + # Here's some real nastiness: Another "instance" of the six module might + # be floating around. Therefore, we can't use isinstance() to check for + # the six meta path importer, since the other six instance will have + # inserted an importer with different class. + if (type(importer).__name__ == "_SixMetaPathImporter" and + importer.name == __name__): + del sys.meta_path[i] + break + del i, importer +# Finally, add the importer to the meta path import hook. +sys.meta_path.append(_importer) From c29cb1cba7fa7bf989bb2da91271ff77f1d1e745 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 3 Oct 2017 20:18:59 -0600 Subject: [PATCH 13/41] cleanup --- scripts/Tools/case.qstatus | 2 +- scripts/Tools/case.submit | 2 +- scripts/Tools/check_case | 2 +- scripts/Tools/check_input_data | 2 +- scripts/Tools/check_lockedfiles | 2 +- scripts/Tools/cime_bisect | 2 +- scripts/Tools/compare_namelists | 2 +- scripts/Tools/compare_test_results | 2 +- scripts/Tools/component_compare_baseline | 2 +- scripts/Tools/component_compare_copy | 2 +- scripts/Tools/component_compare_test | 2 +- scripts/Tools/component_generate_baseline | 2 +- scripts/Tools/getTiming | 2 +- scripts/Tools/jenkins_generic_job | 2 +- scripts/Tools/list_acme_tests | 2 +- scripts/Tools/normalize_cases | 2 +- scripts/Tools/pelayout | 2 +- scripts/Tools/preview_namelists | 2 +- scripts/Tools/preview_run | 2 +- scripts/Tools/save_provenance | 2 +- scripts/Tools/simple_compare | 2 +- scripts/Tools/testreporter.py | 2 +- scripts/Tools/update_acme_tests | 2 +- scripts/Tools/wait_for_tests | 2 +- scripts/Tools/xmlchange | 2 +- scripts/Tools/xmlquery | 2 +- scripts/lib/CIME/case.py | 1 + scripts/lib/CIME/code_checker.py | 4 ++++ 28 files changed, 31 insertions(+), 26 deletions(-) diff --git a/scripts/Tools/case.qstatus b/scripts/Tools/case.qstatus index 748addf7e90..9d64209e14e 100755 --- a/scripts/Tools/case.qstatus +++ b/scripts/Tools/case.qstatus @@ -4,7 +4,7 @@ Show the batch status of all jobs associated with this case. """ -from .standard_script_setup import * +from standard_script_setup import * from CIME.case import Case from CIME.test_status import * diff --git a/scripts/Tools/case.submit b/scripts/Tools/case.submit index 71c69cd4777..fc1787680bf 100755 --- a/scripts/Tools/case.submit +++ b/scripts/Tools/case.submit @@ -4,7 +4,7 @@ case.submit - Submit a cesm workflow to the queueing system or run it if there is no queueing system. A cesm workflow may include multiple jobs. """ -from .standard_script_setup import * +from standard_script_setup import * from CIME.case_submit import submit from CIME.case import Case diff --git a/scripts/Tools/check_case b/scripts/Tools/check_case index c32cf928e9c..07968a75ecb 100755 --- a/scripts/Tools/check_case +++ b/scripts/Tools/check_case @@ -4,7 +4,7 @@ check_case verifies that the case is set up correctly """ -from .standard_script_setup import * +from standard_script_setup import * from CIME.utils import expect from CIME.case import Case diff --git a/scripts/Tools/check_input_data b/scripts/Tools/check_input_data index 78cf183ac78..811920ade56 100755 --- a/scripts/Tools/check_input_data +++ b/scripts/Tools/check_input_data @@ -5,7 +5,7 @@ Check (and optionally download) input data files. Should be run from case. """ -from .standard_script_setup import * +from standard_script_setup import * from CIME.utils import get_model from CIME.check_input_data import check_input_data, SVN_LOCS diff --git a/scripts/Tools/check_lockedfiles b/scripts/Tools/check_lockedfiles index c327bf9b558..852ec72c20e 100755 --- a/scripts/Tools/check_lockedfiles +++ b/scripts/Tools/check_lockedfiles @@ -3,7 +3,7 @@ This script compares xml files """ -from .standard_script_setup import * +from standard_script_setup import * from CIME.check_lockedfiles import check_lockedfiles from CIME.case import Case diff --git a/scripts/Tools/cime_bisect b/scripts/Tools/cime_bisect index e8871e51de7..23d3dde1ecc 100755 --- a/scripts/Tools/cime_bisect +++ b/scripts/Tools/cime_bisect @@ -5,7 +5,7 @@ A script to help track down the commit that caused a test to fail. This script is intended to be run by git bisect. """ -from .standard_script_setup import * +from standard_script_setup import * from CIME.utils import expect, run_cmd_no_fail, run_cmd from CIME.XML.machines import Machines diff --git a/scripts/Tools/compare_namelists b/scripts/Tools/compare_namelists index f8b99b5ef06..03ca3c808d4 100755 --- a/scripts/Tools/compare_namelists +++ b/scripts/Tools/compare_namelists @@ -5,7 +5,7 @@ Compare namelists. Should be called by an ACME test. Designed to not be sensitive to order or whitespace. """ -from .standard_script_setup import * +from standard_script_setup import * import CIME.compare_namelists from CIME.utils import expect diff --git a/scripts/Tools/compare_test_results b/scripts/Tools/compare_test_results index 2f2e0579f45..91967961a71 100755 --- a/scripts/Tools/compare_test_results +++ b/scripts/Tools/compare_test_results @@ -21,7 +21,7 @@ You may need to load modules for cprnc to work. """ -from .standard_script_setup import * +from standard_script_setup import * from CIME.XML.machines import Machines from CIME.compare_test_results import compare_test_results diff --git a/scripts/Tools/component_compare_baseline b/scripts/Tools/component_compare_baseline index 46a6bf36f9d..970d12600d5 100755 --- a/scripts/Tools/component_compare_baseline +++ b/scripts/Tools/component_compare_baseline @@ -4,7 +4,7 @@ Compares current component history files against baselines """ -from .standard_script_setup import * +from standard_script_setup import * from CIME.case import Case from CIME.hist_utils import compare_baseline diff --git a/scripts/Tools/component_compare_copy b/scripts/Tools/component_compare_copy index 0ccbd975c7a..d7c47b09b96 100755 --- a/scripts/Tools/component_compare_copy +++ b/scripts/Tools/component_compare_copy @@ -5,7 +5,7 @@ Copy the most recent batch of hist files in a case, adding the given suffix. This allows us to save these results if we want to run the case again. """ -from .standard_script_setup import * +from standard_script_setup import * from CIME.case import Case from CIME.hist_utils import copy diff --git a/scripts/Tools/component_compare_test b/scripts/Tools/component_compare_test index bfdb885919b..0af20a07b15 100755 --- a/scripts/Tools/component_compare_test +++ b/scripts/Tools/component_compare_test @@ -4,7 +4,7 @@ Compares two component history files in the testcase directory """ -from .standard_script_setup import * +from standard_script_setup import * from CIME.case import Case from CIME.hist_utils import compare_test diff --git a/scripts/Tools/component_generate_baseline b/scripts/Tools/component_generate_baseline index 547404a8a5f..9268ab90acf 100755 --- a/scripts/Tools/component_generate_baseline +++ b/scripts/Tools/component_generate_baseline @@ -4,7 +4,7 @@ Copies current component history files into baselines """ -from .standard_script_setup import * +from standard_script_setup import * from CIME.case import Case from CIME.hist_utils import generate_baseline diff --git a/scripts/Tools/getTiming b/scripts/Tools/getTiming index a3efa3e7ad3..22878582ee5 100755 --- a/scripts/Tools/getTiming +++ b/scripts/Tools/getTiming @@ -3,7 +3,7 @@ Get timing information from run """ -from .standard_script_setup import * +from standard_script_setup import * import argparse, sys, os from CIME.case import Case from CIME.get_timing import get_timing diff --git a/scripts/Tools/jenkins_generic_job b/scripts/Tools/jenkins_generic_job index 7870ecf5ae8..1331a4dbced 100755 --- a/scripts/Tools/jenkins_generic_job +++ b/scripts/Tools/jenkins_generic_job @@ -7,7 +7,7 @@ wait_for_tests that handles cleanup of old test results and ensures that the batch system is left in a clean state. """ -from .standard_script_setup import * +from standard_script_setup import * import CIME.wait_for_tests from CIME.utils import expect diff --git a/scripts/Tools/list_acme_tests b/scripts/Tools/list_acme_tests index 46f25f21510..a0ab818c13d 100755 --- a/scripts/Tools/list_acme_tests +++ b/scripts/Tools/list_acme_tests @@ -5,7 +5,7 @@ List acme test suites. Can be used to show what's being tested. Can just list tested grids, compsets, etc. """ -from .standard_script_setup import * +from standard_script_setup import * from CIME.utils import expect import update_acme_tests diff --git a/scripts/Tools/normalize_cases b/scripts/Tools/normalize_cases index 68395807b9e..848077ee731 100755 --- a/scripts/Tools/normalize_cases +++ b/scripts/Tools/normalize_cases @@ -8,7 +8,7 @@ This is for debugging purposes and meant to assist the user when they want to run case_diff. """ -from .standard_script_setup import * +from standard_script_setup import * from CIME.utils import expect, run_cmd_no_fail import argparse, sys, os, doctest, glob diff --git a/scripts/Tools/pelayout b/scripts/Tools/pelayout index abb60f04168..6422e450287 100755 --- a/scripts/Tools/pelayout +++ b/scripts/Tools/pelayout @@ -28,7 +28,7 @@ information is available at https://github.com/ESMCI/cime. it under the same terms as Python itself. """ -from .standard_script_setup import * +from standard_script_setup import * from CIME.case import Case from CIME.utils import expect, convert_to_string diff --git a/scripts/Tools/preview_namelists b/scripts/Tools/preview_namelists index 491c47d94a3..4bece0ee573 100755 --- a/scripts/Tools/preview_namelists +++ b/scripts/Tools/preview_namelists @@ -6,7 +6,7 @@ which creates the namelist and other model input files, it then copies those files to the CaseDocs subdirectory for inspection """ -from .standard_script_setup import * +from standard_script_setup import * from CIME.preview_namelists import create_namelists from CIME.case import Case diff --git a/scripts/Tools/preview_run b/scripts/Tools/preview_run index 91deb961376..c4eeade3216 100755 --- a/scripts/Tools/preview_run +++ b/scripts/Tools/preview_run @@ -16,7 +16,7 @@ Example: ./xmlchange --subgroup=case.run BATCH_COMMAND_FLAGS='--some-flag --other-flag' """ -from .standard_script_setup import * +from standard_script_setup import * from CIME.case import Case diff --git a/scripts/Tools/save_provenance b/scripts/Tools/save_provenance index 039867c7061..ce68a4d5ac2 100755 --- a/scripts/Tools/save_provenance +++ b/scripts/Tools/save_provenance @@ -4,7 +4,7 @@ This tool provide command-line access to provenance-saving functionality """ -from .standard_script_setup import * +from standard_script_setup import * from CIME.case import Case from CIME.provenance import * diff --git a/scripts/Tools/simple_compare b/scripts/Tools/simple_compare index 99767bf8e6f..d005d112df1 100755 --- a/scripts/Tools/simple_compare +++ b/scripts/Tools/simple_compare @@ -5,7 +5,7 @@ Compare files in a normalized way. Used by create_test for diffing non-namelist files. """ -from .standard_script_setup import * +from standard_script_setup import * import CIME.simple_compare from CIME.utils import expect diff --git a/scripts/Tools/testreporter.py b/scripts/Tools/testreporter.py index 2f87cf97056..41f73bf0372 100755 --- a/scripts/Tools/testreporter.py +++ b/scripts/Tools/testreporter.py @@ -6,7 +6,7 @@ """ -from .standard_script_setup import * +from standard_script_setup import * from CIME.XML.env_build import EnvBuild from CIME.XML.env_case import EnvCase diff --git a/scripts/Tools/update_acme_tests b/scripts/Tools/update_acme_tests index 38c32ee5ccc..4a6cdaec141 100755 --- a/scripts/Tools/update_acme_tests +++ b/scripts/Tools/update_acme_tests @@ -8,7 +8,7 @@ this file to create new test entries, and then inserts these entries into the XML file. """ -from .standard_script_setup import * +from standard_script_setup import * from CIME.utils import expect from update_acme_tests import get_test_suites, update_acme_tests diff --git a/scripts/Tools/wait_for_tests b/scripts/Tools/wait_for_tests index 471126979e4..fa97f818c16 100755 --- a/scripts/Tools/wait_for_tests +++ b/scripts/Tools/wait_for_tests @@ -8,7 +8,7 @@ for the RUN phase specifically and will not terminate if the RUN phase didn't happen. """ -from .standard_script_setup import * +from standard_script_setup import * import CIME.wait_for_tests diff --git a/scripts/Tools/xmlchange b/scripts/Tools/xmlchange index fdfdb712896..55d4b897d2c 100755 --- a/scripts/Tools/xmlchange +++ b/scripts/Tools/xmlchange @@ -10,7 +10,7 @@ """ -from .standard_script_setup import * +from standard_script_setup import * from CIME.utils import expect, convert_to_type, append_case_status from CIME.case import Case diff --git a/scripts/Tools/xmlquery b/scripts/Tools/xmlquery index 8bc9ecd33c2..e0ccf61b812 100755 --- a/scripts/Tools/xmlquery +++ b/scripts/Tools/xmlquery @@ -17,7 +17,7 @@ If this tool is missing any feature that you need, please go to github and creat it under the same terms as Python itself. """ -from .standard_script_setup import * +from standard_script_setup import * from CIME.case import Case from CIME.utils import expect, convert_to_string diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index f3162c1865b..13f9683c2ba 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -7,6 +7,7 @@ from copy import deepcopy import glob, os, shutil, math from CIME.XML.standard_module_setup import * +#pylint: disable=import-error,redefined-builtin from CIME.six.moves import input from CIME.utils import expect, get_cime_root, append_status from CIME.utils import convert_to_type, get_model, get_project diff --git a/scripts/lib/CIME/code_checker.py b/scripts/lib/CIME/code_checker.py index 6aae473c5d4..8eb18f25f76 100644 --- a/scripts/lib/CIME/code_checker.py +++ b/scripts/lib/CIME/code_checker.py @@ -100,6 +100,10 @@ def check_code(files, num_procs=10, interactive=False): # Check every python file files_to_check = get_all_checkable_files() + if "scripts/lib/CIME/six.py" in files_to_check: + files_to_check.remove("scripts/lib/CIME/six.py") + logger.info("Not checking contributed file six.py") + expect(len(files_to_check) > 0, "No matching files found") # No point in using more threads than files From a22abb6eaa2f47f13e1cf790b8cdf748148e56b9 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 3 Oct 2017 20:54:41 -0600 Subject: [PATCH 14/41] fix configparser --- scripts/lib/CIME/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/lib/CIME/utils.py b/scripts/lib/CIME/utils.py index 5dc1d2f4180..aae924bb986 100644 --- a/scripts/lib/CIME/utils.py +++ b/scripts/lib/CIME/utils.py @@ -7,6 +7,8 @@ import warnings from CIME import six from contextlib import contextmanager +#pylint: disable=import-error +from CIME.six.moves import configparser # Return this error code if the scripts worked but tests failed TESTS_FAILED_ERR_CODE = 100 @@ -87,11 +89,10 @@ def _read_cime_config_file(): CIME_MODEL=acme,cesm PROJECT=someprojectnumber """ - from configparser import SafeConfigParser as config_parser cime_config_file = os.path.abspath(os.path.join(os.path.expanduser("~"), ".cime","config")) - cime_config = config_parser() + cime_config = configparser.SafeConfigParser() if(os.path.isfile(cime_config_file)): cime_config.read(cime_config_file) else: From f04afb587df47c7a06cfb1c5c464ed3892815677 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 3 Oct 2017 21:48:55 -0600 Subject: [PATCH 15/41] fix pylint issues --- scripts/Tools/case_diff | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/Tools/case_diff b/scripts/Tools/case_diff index ae5dac241ee..2d17ccee05c 100755 --- a/scripts/Tools/case_diff +++ b/scripts/Tools/case_diff @@ -68,8 +68,8 @@ def recursive_diff(dir1, dir2, repls, show_binary=False, skip_list=()): for dirname, set_obj in [(dir1, dir1_only), (dir2, dir2_only)]: for item in sorted(set_obj): if (item not in skip_list): - print "===============================================================================" - print os.path.join(dirname, item), "is unique" + print ("===============================================================================") + print (os.path.join(dirname, item), "is unique") num_differing_files += 1 # Handling of the common items is trickier @@ -82,8 +82,8 @@ def recursive_diff(dir1, dir2, repls, show_binary=False, skip_list=()): # If the directory status of the files differs, report diff if (path1isdir != os.path.isdir(path2)): - print "===============================================================================" - print path1 + " DIFFERS (directory status)" + print ("===============================================================================") + print (path1 + " DIFFERS (directory status)") num_differing_files += 1 continue @@ -111,10 +111,10 @@ def recursive_diff(dir1, dir2, repls, show_binary=False, skip_list=()): stat, out, _ = run_cmd("diff -w {} -".format(path1), input_str=the_text) if (stat != 0): - print "===============================================================================" - print path1 + " DIFFERS (contents)" + print ("===============================================================================") + print (path1 + " DIFFERS (contents)") num_differing_files += 1 - print " ", out + print (" "+ out) return num_differing_files @@ -135,7 +135,7 @@ def _main_func(description): repls[val2] = val1 num_differing_files = recursive_diff(case1, case2, repls, show_binary, skip_list) - print num_differing_files, "files are different" + print (num_differing_files, "files are different") sys.exit(0 if num_differing_files == 0 else 1) ############################################################################### From 170bba5fb57ec899fb2b5e2cb5e2b84cacfff571 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 3 Oct 2017 21:49:12 -0600 Subject: [PATCH 16/41] python3 fixes --- scripts/Tools/case.build | 2 +- scripts/lib/CIME/BuildTools/configure.py | 2 +- .../lib/CIME/BuildTools/macrowriterbase.py | 4 +-- scripts/lib/CIME/XML/generic_xml.py | 2 +- scripts/lib/CIME/utils.py | 2 +- scripts/lib/update_acme_tests.py | 8 ++++-- scripts/tests/scripts_regression_tests.py | 28 +++++++++---------- 7 files changed, 26 insertions(+), 22 deletions(-) diff --git a/scripts/Tools/case.build b/scripts/Tools/case.build index c8ec78096c0..4ac984e3745 100755 --- a/scripts/Tools/case.build +++ b/scripts/Tools/case.build @@ -84,7 +84,7 @@ def _main_func(description): if cleanlist is not None or clean_all: build.clean(case, cleanlist, clean_all) elif(testname is not None): - logging.warn("Building test for {} in directory {}".format(testname, + logging.warning("Building test for {} in directory {}".format(testname, caseroot)) try: # The following line can throw exceptions if the testname is diff --git a/scripts/lib/CIME/BuildTools/configure.py b/scripts/lib/CIME/BuildTools/configure.py index 90825dde97d..81bb3c331bc 100644 --- a/scripts/lib/CIME/BuildTools/configure.py +++ b/scripts/lib/CIME/BuildTools/configure.py @@ -73,7 +73,7 @@ def _generate_env_mach_specific(output_dir, machobj, compiler, mpilib, debug, """ ems_path = os.path.join(output_dir, "env_mach_specific.xml") if os.path.exists(ems_path): - logger.warn("{} already exists, delete to replace".format(ems_path)) + logger.warning("{} already exists, delete to replace".format(ems_path)) return ems_file = EnvMachSpecific(output_dir, unit_testing=unit_testing) ems_file.populate(machobj) diff --git a/scripts/lib/CIME/BuildTools/macrowriterbase.py b/scripts/lib/CIME/BuildTools/macrowriterbase.py index 8061d6414a2..0147a56de96 100644 --- a/scripts/lib/CIME/BuildTools/macrowriterbase.py +++ b/scripts/lib/CIME/BuildTools/macrowriterbase.py @@ -133,7 +133,7 @@ def write_line(self, line): A trailing newline is added, whether or not the input has one. """ - self.output.write(unicode(self.indent_string() + line + "\n")) + self.output.write(str(self.indent_string() + line + "\n")) @abstractmethod def environment_variable_string(self, name): @@ -208,7 +208,7 @@ def write_macros_file_v1(macros, compiler, os_, machine, macros_file="Macros", o # A few things can be used from environ if not in XML for item in ["MPI_PATH", "NETCDF_PATH"]: if not item in macros and item in os.environ: - logger.warn("Setting {} from Environment".format(item)) + logger.warning("Setting {} from Environment".format(item)) macros[item] = os.environ[item] with open(macros_file, "w") as fd: diff --git a/scripts/lib/CIME/XML/generic_xml.py b/scripts/lib/CIME/XML/generic_xml.py index 6ad497770a6..fc02c2459a8 100644 --- a/scripts/lib/CIME/XML/generic_xml.py +++ b/scripts/lib/CIME/XML/generic_xml.py @@ -267,7 +267,7 @@ def validate_xml_file(self, filename, schema): logger.debug("Checking file {} against schema {}".format(filename, schema)) run_cmd_no_fail("{} --noout --schema {} {}".format(xmllint, schema, filename)) else: - logger.warn("xmllint not found, could not validate file {}".format(filename)) + logger.warning("xmllint not found, could not validate file {}".format(filename)) def get_element_text(self, element_name, attributes=None, root=None, xpath=None): element_node = self.get_optional_node(element_name, attributes, root, xpath) diff --git a/scripts/lib/CIME/utils.py b/scripts/lib/CIME/utils.py index aae924bb986..f75085ea240 100644 --- a/scripts/lib/CIME/utils.py +++ b/scripts/lib/CIME/utils.py @@ -157,7 +157,7 @@ def get_model(): >>> del os.environ["CIME_MODEL"] >>> set_model('rocky') >>> get_model() - u'rocky' + 'rocky' >>> reset_cime_config() """ model = os.environ.get("CIME_MODEL") diff --git a/scripts/lib/update_acme_tests.py b/scripts/lib/update_acme_tests.py index c539cb01bdc..87a903a52ac 100644 --- a/scripts/lib/update_acme_tests.py +++ b/scripts/lib/update_acme_tests.py @@ -355,11 +355,15 @@ def sort_by_time(test_one, test_two): """ rec1, rec2 = get_recommended_test_time(test_one), get_recommended_test_time(test_two) if rec1 == rec2: - return cmp(test_one, test_two) + return (test_one > test_two) - (test_two < test_one) else: if rec2 is None: return -1 elif rec1 is None: return 1 else: - return cmp(convert_to_seconds(rec2), convert_to_seconds(rec1)) + a = convert_to_seconds(rec2) + b = convert_to_seconds(rec1) + return (a < b) - (b < a) + + diff --git a/scripts/tests/scripts_regression_tests.py b/scripts/tests/scripts_regression_tests.py index 0be3bcd5e87..32aeab67201 100755 --- a/scripts/tests/scripts_regression_tests.py +++ b/scripts/tests/scripts_regression_tests.py @@ -1,7 +1,7 @@ #!/usr/bin/env python import io, glob, os, re, shutil, signal, sys, tempfile, \ - threading, time, logging, unittest, getpass, string + threading, time, logging, unittest, getpass from xml.etree.ElementTree import ParseError @@ -340,7 +340,7 @@ def test_d_create_clone_new_user(self): run_cmd_assert_result(self, "./xmlchange USER=this_is_not_a_user", from_dir=prevtestdir) - fakeoutputroot = string.replace(cls._testroot, os.environ.get("USER"), "this_is_not_a_user") + fakeoutputroot = cls._testroot.replace(os.environ.get("USER"), "this_is_not_a_user") run_cmd_assert_result(self, "./xmlchange CIME_OUTPUT_ROOT=%s"%fakeoutputroot, from_dir=prevtestdir) @@ -1692,7 +1692,7 @@ def get_macros(macro_maker, build_xml, build_system): """ # Build.write_macros expects file-like objects as input, so # we need to wrap the strings in StringIO objects. - xml = io.StringIO(unicode(build_xml)) + xml = io.StringIO(str(build_xml)) output = io.StringIO() output_format = None if build_system == "Makefile": @@ -1938,7 +1938,7 @@ def test_script_rejects_bad_build_system(self): """The macro writer rejects a bad build system string.""" maker = Compilers(MockMachines("mymachine", "SomeOS"), version="2.0") bad_string = "argle-bargle." - with self.assertRaisesRegexp( + with self.assertRaisesRegex( SystemExit, "Unrecognized build system provided to write_macros: " + bad_string): get_macros(maker, "This string is irrelevant.", bad_string) @@ -2040,7 +2040,7 @@ def test_reject_duplicate_defaults(self): """The macro writer dies if given many defaults.""" xml1 = """/path/to/default""" xml2 = """/path/to/other_default""" - with self.assertRaisesRegexp( + with self.assertRaisesRegex( SystemExit, "Variable MPI_PATH is set ambiguously in config_build.xml."): self.xml_to_tester(xml1+xml2) @@ -2049,7 +2049,7 @@ def test_reject_duplicates(self): """The macro writer dies if given many matches for a given configuration.""" xml1 = """/path/to/mpich""" xml2 = """/path/to/mpich2""" - with self.assertRaisesRegexp( + with self.assertRaisesRegex( SystemExit, "Variable MPI_PATH is set ambiguously in config_build.xml."): self.xml_to_tester(xml1+xml2) @@ -2058,7 +2058,7 @@ def test_reject_ambiguous(self): """The macro writer dies if given an ambiguous set of matches.""" xml1 = """/path/to/mpich""" xml2 = """/path/to/mpi-debug""" - with self.assertRaisesRegexp( + with self.assertRaisesRegex( SystemExit, "Variable MPI_PATH is set ambiguously in config_build.xml."): self.xml_to_tester(xml1+xml2) @@ -2190,7 +2190,7 @@ def test_env_and_shell_command(self): tester.assert_variable_equals("FFLAGS", "-O2 -fast", env={"OPT_LEVEL": "2"}) xml1 = """-O$SHELL{echo $ENV{OPT_LEVEL}} -fast""" err_msg = "Nesting not allowed.*" - with self.assertRaisesRegexp(SystemExit, err_msg): + with self.assertRaisesRegex(SystemExit, err_msg): self.xml_to_tester(xml1) def test_config_variable_insertion(self): @@ -2220,12 +2220,12 @@ def test_config_reject_self_references(self): # references. xml1 = """MPI_LIB_NAME""" err_msg = ".* has bad references." - with self.assertRaisesRegexp(SystemExit, err_msg): + with self.assertRaisesRegex(SystemExit, err_msg): self.xml_to_tester(""+xml1+"") xml1 = """${MPI_LIB_NAME}""" err_msg = ".* has bad references." - with self.assertRaisesRegexp(SystemExit, err_msg): + with self.assertRaisesRegex(SystemExit, err_msg): self.xml_to_tester(""+xml1+"") def test_config_reject_cyclical_references(self): @@ -2233,12 +2233,12 @@ def test_config_reject_cyclical_references(self): xml1 = """MPI_PATH""" xml2 = """MPI_LIB_NAME""" err_msg = ".* has bad references." - with self.assertRaisesRegexp(SystemExit, err_msg): + with self.assertRaisesRegex(SystemExit, err_msg): self.xml_to_tester(""+xml1+xml2+"") xml1 = """${MPI_PATH}""" xml2 = """${MPI_LIB_NAME}""" err_msg = ".* has bad references." - with self.assertRaisesRegexp(SystemExit, err_msg): + with self.assertRaisesRegex(SystemExit, err_msg): self.xml_to_tester(""+xml1+xml2+"") def test_variable_insertion_with_machine_specific_setting(self): @@ -2247,14 +2247,14 @@ def test_variable_insertion_with_machine_specific_setting(self): xml2 = """MPI_PATH""".format(self.test_machine) xml3 = """MPI_LIB_NAME""" err_msg = ".* has bad references." - with self.assertRaisesRegexp(SystemExit, err_msg): + with self.assertRaisesRegex(SystemExit, err_msg): self.xml_to_tester(xml1+xml2+xml3) xml1 = """something""" xml2 = """MPI_PATH""".format(self.test_machine) xml3 = """${MPI_LIB_NAME}""" err_msg = ".* has bad references." - with self.assertRaisesRegexp(SystemExit, err_msg): + with self.assertRaisesRegex(SystemExit, err_msg): self.xml_to_tester(xml1+xml2+xml3) From 13b8821e7927da340bfa76d763faf97c2ffd8bb2 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 3 Oct 2017 21:57:33 -0600 Subject: [PATCH 17/41] replace warn with warning --- scripts/Tools/case_diff | 2 +- scripts/lib/CIME/build.py | 4 ++-- scripts/lib/CIME/buildnml.py | 2 +- scripts/lib/CIME/case.py | 16 ++++++++-------- scripts/lib/CIME/case_clone.py | 6 +++--- scripts/lib/CIME/case_run.py | 8 ++++---- scripts/lib/CIME/case_st_archive.py | 6 +++--- scripts/lib/CIME/case_submit.py | 2 +- scripts/lib/CIME/hist_utils.py | 2 +- scripts/lib/CIME/nmlgen.py | 2 +- scripts/lib/CIME/preview_namelists.py | 2 +- scripts/lib/CIME/test_scheduler.py | 2 +- scripts/lib/CIME/user_mod_support.py | 6 +++--- scripts/lib/CIME/utils.py | 8 ++++---- 14 files changed, 34 insertions(+), 34 deletions(-) diff --git a/scripts/Tools/case_diff b/scripts/Tools/case_diff index 2d17ccee05c..6568632afb1 100755 --- a/scripts/Tools/case_diff +++ b/scripts/Tools/case_diff @@ -100,7 +100,7 @@ def recursive_diff(dir1, dir2, repls, show_binary=False, skip_list=()): stat, out, err = run_cmd("file {}".format(path1)) if (stat != 0): - logging.warn("Failed to probe file '{}', out: '{}', err: '{}'".format(path1, out, err)) + logging.warning("Failed to probe file '{}', out: '{}', err: '{}'".format(path1, out, err)) continue is_text_file = "text" in out diff --git a/scripts/lib/CIME/build.py b/scripts/lib/CIME/build.py index b46b9effa8c..3d9857e21fe 100644 --- a/scripts/lib/CIME/build.py +++ b/scripts/lib/CIME/build.py @@ -252,7 +252,7 @@ def _build_libraries(case, exeroot, sharedpath, caseroot, cimeroot, libroot, lid bldlog = open(file_build, "r") for line in bldlog: if re.search("Current setting for", line): - logger.warn(line) + logger.warning(line) # clm not a shared lib for ACME if get_model() != "acme" and (buildlist is None or "lnd" in buildlist): @@ -290,7 +290,7 @@ def _build_model_thread(config_dir, compclass, compname, caseroot, libroot, bldr t1 = time.time() cmd = os.path.join(caseroot, "SourceMods", "src." + compname, "buildlib") if os.path.isfile(cmd): - logger.warn("WARNING: using local buildlib script for {}".format(compname)) + logger.warning("WARNING: using local buildlib script for {}".format(compname)) else: cmd = os.path.join(config_dir, "buildlib") expect(os.path.isfile(cmd), "Could not find buildlib for {}".format(compname)) diff --git a/scripts/lib/CIME/buildnml.py b/scripts/lib/CIME/buildnml.py index a600f681432..85c4f5b96ce 100644 --- a/scripts/lib/CIME/buildnml.py +++ b/scripts/lib/CIME/buildnml.py @@ -98,7 +98,7 @@ def create_namelist_infile(case, user_nl_file, namelist_infile, infile_text=""): with open(user_nl_file, "r") as file_usernl: lines_input = file_usernl.readlines() else: - logger.warn("WARNING: No file {} found in case directory".format(user_nl_file)) + logger.warning("WARNING: No file {} found in case directory".format(user_nl_file)) lines_output = [] lines_output.append("&comp_inparm \n") diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index 13f9683c2ba..ddeeb87ef8b 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -404,7 +404,7 @@ def set_valid_values(self, item, valid_values): def set_lookup_value(self, item, value): if item in list(self.lookups.keys()) and self.lookups[item] is not None: - logger.warn("Item {} already in lookups with value {}".format(item,self.lookups[item])) + logger.warning("Item {} already in lookups with value {}".format(item,self.lookups[item])) else: logger.debug("Setting in lookups: item {}, value {}".format(item,value)) self.lookups[item] = value @@ -1244,7 +1244,7 @@ def set_model_version(self, model): if version != "unknown": logger.info("{} model version found: {}".format(model, version)) else: - logger.warn("WARNING: No {} Model version found.".format(model)) + logger.warning("WARNING: No {} Model version found.".format(model)) def load_env(self): if not self._is_env_loaded: @@ -1285,9 +1285,9 @@ def _check_testlists(self, compset_alias, grid_name, files): if test["category"] == "prealpha" or test["category"] == "prebeta" or "aux_" in test["category"]: testcnt += 1 if testcnt > 0: - logger.warn("\n*********************************************************************************************************************************") - logger.warn("This compset and grid combination is not scientifically supported, however it is used in {:d} tests.".format(testcnt)) - logger.warn("*********************************************************************************************************************************\n") + logger.warning("\n*********************************************************************************************************************************") + logger.warning("This compset and grid combination is not scientifically supported, however it is used in {:d} tests.".format(testcnt)) + logger.warning("*********************************************************************************************************************************\n") else: expect(False, "\nThis compset and grid combination is untested in CESM. " "Override this warning with the --run-unsupported option to create_newcase.", @@ -1305,7 +1305,7 @@ def set_file(self, xmlfile): gfile = GenericXML(infile=xmlfile) ftype = gfile.get_id() components = self.get_value("COMP_CLASSES") - logger.warn("setting case file to {}".format(xmlfile)) + logger.warning("setting case file to {}".format(xmlfile)) new_env_file = None for env_file in self._env_entryid_files: if os.path.basename(env_file.filename) == ftype: @@ -1412,10 +1412,10 @@ def create(self, casename, srcroot, compset_name, grid_name, except: if os.path.exists(self._caseroot): if not logger.isEnabledFor(logging.DEBUG) and not test: - logger.warn("Failed to setup case, removing {}\nUse --debug to force me to keep caseroot".format(self._caseroot)) + logger.warning("Failed to setup case, removing {}\nUse --debug to force me to keep caseroot".format(self._caseroot)) shutil.rmtree(self._caseroot) else: - logger.warn("Leaving broken case dir {}".format(self._caseroot)) + logger.warning("Leaving broken case dir {}".format(self._caseroot)) raise diff --git a/scripts/lib/CIME/case_clone.py b/scripts/lib/CIME/case_clone.py index 0b50818501f..9fb0d63f5d9 100644 --- a/scripts/lib/CIME/case_clone.py +++ b/scripts/lib/CIME/case_clone.py @@ -68,8 +68,8 @@ def create_case_clone(case, newcase, keepexe=False, mach_dir=None, project=None, newcase.set_value("BUILD_COMPLETE","TRUE") orig_bld_complete = case.get_value("BUILD_COMPLETE") if not orig_bld_complete: - logger.warn("\nWARNING: Creating a clone with --keepexe before building the original case may cause PIO_TYPENAME to be invalid in the clone") - logger.warn("Avoid this message by building case one before you clone.\n") + logger.warning("\nWARNING: Creating a clone with --keepexe before building the original case may cause PIO_TYPENAME to be invalid in the clone") + logger.warning("Avoid this message by building case one before you clone.\n") else: newcase.set_value("BUILD_COMPLETE","FALSE") @@ -133,7 +133,7 @@ def create_case_clone(case, newcase, keepexe=False, mach_dir=None, project=None, success, comment = compare_files(os.path.join(newcaseroot, "env_build.xml"), os.path.join(newcaseroot, "LockedFiles", "env_build.xml")) if not success: - logger.warn(comment) + logger.warning(comment) shutil.rmtree(newcase_root) expect(False, "env_build.xml cannot be changed via usermods if keepexe is an option: \n " "Failed to clone case, removed {}\n".format(newcase_root)) diff --git a/scripts/lib/CIME/case_run.py b/scripts/lib/CIME/case_run.py index 0c657c7e5e8..0b5f756ff8f 100644 --- a/scripts/lib/CIME/case_run.py +++ b/scripts/lib/CIME/case_run.py @@ -210,12 +210,12 @@ def resubmit_check(case): # Note that Mira requires special logic dout_s = case.get_value("DOUT_S") - logger.warn("dout_s {} ".format(dout_s)) + logger.warning("dout_s {} ".format(dout_s)) mach = case.get_value("MACH") - logger.warn("mach {} ".format(mach)) + logger.warning("mach {} ".format(mach)) testcase = case.get_value("TESTCASE") resubmit_num = case.get_value("RESUBMIT") - logger.warn("resubmit_num {}".format(resubmit_num)) + logger.warning("resubmit_num {}".format(resubmit_num)) # If dout_s is True than short-term archiving handles the resubmit # If dout_s is True and machine is mira submit the st_archive script resubmit = False @@ -310,7 +310,7 @@ def case_run(case, skip_pnl=False): save_postrun_provenance(case) - logger.warn("check for resubmit") + logger.warning("check for resubmit") resubmit_check(case) return True diff --git a/scripts/lib/CIME/case_st_archive.py b/scripts/lib/CIME/case_st_archive.py index 9c756d93a5d..ea8931a88a6 100644 --- a/scripts/lib/CIME/case_st_archive.py +++ b/scripts/lib/CIME/case_st_archive.py @@ -392,9 +392,9 @@ def _archive_restarts_date_comp(case, archive, archive_entry, try: os.remove(srcfile) except OSError: - logger.warn("unable to remove interim restart file {}".format(srcfile)) + logger.warning("unable to remove interim restart file {}".format(srcfile)) else: - logger.warn("interim restart file {} does not exist".format(srcfile)) + logger.warning("interim restart file {} does not exist".format(srcfile)) return histfiles_savein_rundir @@ -512,7 +512,7 @@ def case_st_archive(case, last_date=None, archive_incomplete_logs=True, copy_onl rest_n = case.get_value('REST_N') stop_n = case.get_value('STOP_N') if rest_n < stop_n: - logger.warn('Restart files from end of run will be saved' + logger.warning('Restart files from end of run will be saved' 'interim restart files will be deleted') logger.info("st_archive starting") diff --git a/scripts/lib/CIME/case_submit.py b/scripts/lib/CIME/case_submit.py index 80141ebeb6f..20243b6368e 100644 --- a/scripts/lib/CIME/case_submit.py +++ b/scripts/lib/CIME/case_submit.py @@ -62,7 +62,7 @@ def _submit(case, job=None, resubmit=False, no_batch=False, skip_pnl=False, case.set_value("RUN_WITH_SUBMIT", True) case.flush() - logger.warn("submit_jobs {}".format(job)) + logger.warning("submit_jobs {}".format(job)) job_ids = case.submit_jobs(no_batch=no_batch, job=job, skip_pnl=skip_pnl, mail_user=mail_user, mail_type=mail_type, batch_args=batch_args) diff --git a/scripts/lib/CIME/hist_utils.py b/scripts/lib/CIME/hist_utils.py index 59cdd837a87..a06d5468d5e 100644 --- a/scripts/lib/CIME/hist_utils.py +++ b/scripts/lib/CIME/hist_utils.py @@ -403,7 +403,7 @@ def generate_baseline(case, baseline_dir=None, allow_baseline_overwrite=False): # drop the date so that the name is generic newestcpllogfile = case.get_latest_cpl_log(coupler_log_path=case.get_value("LOGDIR")) if newestcpllogfile is None: - logger.warn("No cpl.log file found in log directory {}".format(case.get_value("LOGDIR"))) + logger.warning("No cpl.log file found in log directory {}".format(case.get_value("LOGDIR"))) else: shutil.copyfile(newestcpllogfile, os.path.join(basegen_dir, "cpl.log.gz")) diff --git a/scripts/lib/CIME/nmlgen.py b/scripts/lib/CIME/nmlgen.py index 4a3fd1c6e06..fe2bc304bff 100644 --- a/scripts/lib/CIME/nmlgen.py +++ b/scripts/lib/CIME/nmlgen.py @@ -570,7 +570,7 @@ def add_default(self, name, value=None, ignore_abs_path=None): continue file_path = self.set_abs_file_path(file_path) if not os.path.exists(file_path): - logger.warn("File not found: {} = {}, will attempt to download in check_input_data phase".format(name, literal)) + logger.warning("File not found: {} = {}, will attempt to download in check_input_data phase".format(name, literal)) current_literals[i] = string_to_character_literal(file_path) current_literals = compress_literal_list(current_literals) diff --git a/scripts/lib/CIME/preview_namelists.py b/scripts/lib/CIME/preview_namelists.py index fff00b22145..8706594d2ec 100644 --- a/scripts/lib/CIME/preview_namelists.py +++ b/scripts/lib/CIME/preview_namelists.py @@ -80,7 +80,7 @@ def create_namelists(case, component=None): # first look in the case SourceMods directory cmd = os.path.join(caseroot, "SourceMods", "src."+compname, "buildnml") if os.path.isfile(cmd): - logger.warn("\nWARNING: Using local buildnml file {}\n".format(cmd)) + logger.warning("\nWARNING: Using local buildnml file {}\n".format(cmd)) else: # otherwise look in the component config_dir cmd = os.path.join(config_dir, "buildnml") diff --git a/scripts/lib/CIME/test_scheduler.py b/scripts/lib/CIME/test_scheduler.py index 16180ed8384..98efb015fe4 100644 --- a/scripts/lib/CIME/test_scheduler.py +++ b/scripts/lib/CIME/test_scheduler.py @@ -547,7 +547,7 @@ def _xml_phase(self, test): # handled in create_newcase continue elif opt.startswith('IOP'): - logger.warn("IOP test option not yet implemented") + logger.warning("IOP test option not yet implemented") else: expect(False, "Could not parse option '{}' ".format(opt)) diff --git a/scripts/lib/CIME/user_mod_support.py b/scripts/lib/CIME/user_mod_support.py index d9eac7e7abb..91bd3d65536 100644 --- a/scripts/lib/CIME/user_mod_support.py +++ b/scripts/lib/CIME/user_mod_support.py @@ -62,7 +62,7 @@ def apply_user_mods(caseroot, user_mods_path, keepexe=None): # We overwrite any existing SourceMods file so that later # include_dirs take precedence over earlier ones if os.path.isfile(case_source_mods): - logger.warn("WARNING: Overwriting existing SourceMods in {}".format(case_source_mods)) + logger.warning("WARNING: Overwriting existing SourceMods in {}".format(case_source_mods)) else: logger.info("Adding SourceMod to case {}".format(case_source_mods)) try: @@ -81,7 +81,7 @@ def apply_user_mods(caseroot, user_mods_path, keepexe=None): # Note that use of xmlchange_cmnds has been deprecated and will soon # be removed altogether, so new tests should rely on shell_commands if shell_commands_file.endswith("xmlchange_cmnds"): - logger.warn("xmlchange_cmnds is deprecated and will be removed " +\ + logger.warning("xmlchange_cmnds is deprecated and will be removed " +\ "in a future release; please rename {} shell_commands".format(shell_commands_file)) with open(shell_commands_file,"r") as fd: new_shell_commands = fd.read().replace("xmlchange","xmlchange --force") @@ -130,6 +130,6 @@ def build_include_dirs_list(user_mods_path, include_dirs=None): if os.path.isabs(newpath): build_include_dirs_list(newpath, include_dirs) else: - logger.warn("Could not resolve path '{}' in file '{}'".format(newpath, include_file)) + logger.warning("Could not resolve path '{}' in file '{}'".format(newpath, include_file)) return include_dirs diff --git a/scripts/lib/CIME/utils.py b/scripts/lib/CIME/utils.py index f75085ea240..818e8e6e722 100644 --- a/scripts/lib/CIME/utils.py +++ b/scripts/lib/CIME/utils.py @@ -77,7 +77,7 @@ def check_name(fullname, additional_chars=None, fullpath=False): name = fullname match = re.search(r"["+re.escape(chars)+"]", name) if match is not None: - logger.warn("Illegal character {} found in name {}".format(match.group(0), name)) + logger.warning("Illegal character {} found in name {}".format(match.group(0), name)) return False return True @@ -1115,7 +1115,7 @@ def transform_vars(text, case=None, subgroup=None, overrides=None, default=None) if "-q {{ queue }}" in text: text = "" else: - logger.warn("Could not replace variable '{}'".format(variable)) + logger.warning("Could not replace variable '{}'".format(variable)) text = text.replace(whole_match, "") return text @@ -1274,9 +1274,9 @@ def analyze_build_log(comp, log, compiler): if re.search(warn_re, line): warncnt += 1 if re.search(error_re, line): - logger.warn(line) + logger.warning(line) if re.search(undefined_re, line): - logger.warn(line) + logger.warning(line) if warncnt > 0: logger.info("Component {} build complete with {} warnings".format(comp, warncnt)) From e36af5d42f874c6ea06ea01596e70e5327cff1dc Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 4 Oct 2017 07:41:07 -0600 Subject: [PATCH 18/41] move six to lib dir --- scripts/Tools/cimeteststatus | 2 +- .../lib/CIME/BuildTools/macrowriterbase.py | 2 +- scripts/lib/CIME/XML/test_reporter.py | 2 +- scripts/lib/CIME/case.py | 2 +- scripts/lib/CIME/code_checker.py | 4 +-- scripts/lib/CIME/utils.py | 4 +-- scripts/lib/CIME/wait_for_tests.py | 2 +- scripts/lib/{CIME => }/six.py | 0 scripts/tests/scripts_regression_tests.py | 27 ++++++++++--------- 9 files changed, 23 insertions(+), 22 deletions(-) rename scripts/lib/{CIME => }/six.py (100%) diff --git a/scripts/Tools/cimeteststatus b/scripts/Tools/cimeteststatus index 1c1daf7ffa4..ce940161d42 100755 --- a/scripts/Tools/cimeteststatus +++ b/scripts/Tools/cimeteststatus @@ -7,7 +7,7 @@ Authors: Jay Shollenberger and Ben Andre """ from standard_script_setup import * #pylint: disable=import-error -from CIME.six.moves import urllib +from six.moves import urllib if sys.hexversion < 0x02070000: print(70 * "*") print("ERROR: {0} requires python >= 2.7.x. ".format(sys.argv[0])) diff --git a/scripts/lib/CIME/BuildTools/macrowriterbase.py b/scripts/lib/CIME/BuildTools/macrowriterbase.py index 0147a56de96..7ccf9975336 100644 --- a/scripts/lib/CIME/BuildTools/macrowriterbase.py +++ b/scripts/lib/CIME/BuildTools/macrowriterbase.py @@ -14,7 +14,7 @@ from abc import ABCMeta, abstractmethod from CIME.XML.standard_module_setup import * from CIME.utils import get_cime_root -from CIME.six import add_metaclass +from six import add_metaclass logger = logging.getLogger(__name__) diff --git a/scripts/lib/CIME/XML/test_reporter.py b/scripts/lib/CIME/XML/test_reporter.py index c711bc63827..560267af005 100644 --- a/scripts/lib/CIME/XML/test_reporter.py +++ b/scripts/lib/CIME/XML/test_reporter.py @@ -3,7 +3,7 @@ """ #pylint: disable=import-error -from CIME.six.moves import urllib +from six.moves import urllib from CIME.XML.standard_module_setup import * from CIME.XML.generic_xml import GenericXML from CIME.utils import expect,get_model diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index ddeeb87ef8b..25c6261f132 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -8,7 +8,7 @@ import glob, os, shutil, math from CIME.XML.standard_module_setup import * #pylint: disable=import-error,redefined-builtin -from CIME.six.moves import input +from six.moves import input from CIME.utils import expect, get_cime_root, append_status from CIME.utils import convert_to_type, get_model, get_project from CIME.utils import get_current_commit, check_name diff --git a/scripts/lib/CIME/code_checker.py b/scripts/lib/CIME/code_checker.py index 8eb18f25f76..237314f1f3b 100644 --- a/scripts/lib/CIME/code_checker.py +++ b/scripts/lib/CIME/code_checker.py @@ -100,8 +100,8 @@ def check_code(files, num_procs=10, interactive=False): # Check every python file files_to_check = get_all_checkable_files() - if "scripts/lib/CIME/six.py" in files_to_check: - files_to_check.remove("scripts/lib/CIME/six.py") + if "scripts/lib/six.py" in files_to_check: + files_to_check.remove("scripts/lib/six.py") logger.info("Not checking contributed file six.py") expect(len(files_to_check) > 0, "No matching files found") diff --git a/scripts/lib/CIME/utils.py b/scripts/lib/CIME/utils.py index 818e8e6e722..98a02b4bc51 100644 --- a/scripts/lib/CIME/utils.py +++ b/scripts/lib/CIME/utils.py @@ -5,10 +5,10 @@ import logging, gzip, sys, os, time, re, shutil, glob, string, random, imp, errno, signal import stat as statlib import warnings -from CIME import six +import six from contextlib import contextmanager #pylint: disable=import-error -from CIME.six.moves import configparser +from six.moves import configparser # Return this error code if the scripts worked but tests failed TESTS_FAILED_ERR_CODE = 100 diff --git a/scripts/lib/CIME/wait_for_tests.py b/scripts/lib/CIME/wait_for_tests.py index 9f2a746a4af..5000bff60df 100644 --- a/scripts/lib/CIME/wait_for_tests.py +++ b/scripts/lib/CIME/wait_for_tests.py @@ -1,5 +1,5 @@ #pylint: disable=import-error -from CIME.six.moves import queue +from six.moves import queue import os, time, threading, socket, signal, distutils.spawn, shutil, glob import logging import xml.etree.ElementTree as xmlet diff --git a/scripts/lib/CIME/six.py b/scripts/lib/six.py similarity index 100% rename from scripts/lib/CIME/six.py rename to scripts/lib/six.py diff --git a/scripts/tests/scripts_regression_tests.py b/scripts/tests/scripts_regression_tests.py index 32aeab67201..2670508c46b 100755 --- a/scripts/tests/scripts_regression_tests.py +++ b/scripts/tests/scripts_regression_tests.py @@ -4,6 +4,7 @@ threading, time, logging, unittest, getpass from xml.etree.ElementTree import ParseError +import six LIB_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)),"..","lib") sys.path.append(LIB_DIR) @@ -1692,8 +1693,8 @@ def get_macros(macro_maker, build_xml, build_system): """ # Build.write_macros expects file-like objects as input, so # we need to wrap the strings in StringIO objects. - xml = io.StringIO(str(build_xml)) - output = io.StringIO() + xml = six.StringIO(str(build_xml)) + output = six.StringIO() output_format = None if build_system == "Makefile": output_format = "make" @@ -1938,7 +1939,7 @@ def test_script_rejects_bad_build_system(self): """The macro writer rejects a bad build system string.""" maker = Compilers(MockMachines("mymachine", "SomeOS"), version="2.0") bad_string = "argle-bargle." - with self.assertRaisesRegex( + with six.assertRaisesRegex(self, SystemExit, "Unrecognized build system provided to write_macros: " + bad_string): get_macros(maker, "This string is irrelevant.", bad_string) @@ -2040,7 +2041,7 @@ def test_reject_duplicate_defaults(self): """The macro writer dies if given many defaults.""" xml1 = """/path/to/default""" xml2 = """/path/to/other_default""" - with self.assertRaisesRegex( + with six.assertRaisesRegex(self, SystemExit, "Variable MPI_PATH is set ambiguously in config_build.xml."): self.xml_to_tester(xml1+xml2) @@ -2049,7 +2050,7 @@ def test_reject_duplicates(self): """The macro writer dies if given many matches for a given configuration.""" xml1 = """/path/to/mpich""" xml2 = """/path/to/mpich2""" - with self.assertRaisesRegex( + with six.assertRaisesRegex(self, SystemExit, "Variable MPI_PATH is set ambiguously in config_build.xml."): self.xml_to_tester(xml1+xml2) @@ -2058,7 +2059,7 @@ def test_reject_ambiguous(self): """The macro writer dies if given an ambiguous set of matches.""" xml1 = """/path/to/mpich""" xml2 = """/path/to/mpi-debug""" - with self.assertRaisesRegex( + with six.assertRaisesRegex(self, SystemExit, "Variable MPI_PATH is set ambiguously in config_build.xml."): self.xml_to_tester(xml1+xml2) @@ -2190,7 +2191,7 @@ def test_env_and_shell_command(self): tester.assert_variable_equals("FFLAGS", "-O2 -fast", env={"OPT_LEVEL": "2"}) xml1 = """-O$SHELL{echo $ENV{OPT_LEVEL}} -fast""" err_msg = "Nesting not allowed.*" - with self.assertRaisesRegex(SystemExit, err_msg): + with six.assertRaisesRegex(self,SystemExit, err_msg): self.xml_to_tester(xml1) def test_config_variable_insertion(self): @@ -2220,12 +2221,12 @@ def test_config_reject_self_references(self): # references. xml1 = """MPI_LIB_NAME""" err_msg = ".* has bad references." - with self.assertRaisesRegex(SystemExit, err_msg): + with six.assertRaisesRegex(self,SystemExit, err_msg): self.xml_to_tester(""+xml1+"") xml1 = """${MPI_LIB_NAME}""" err_msg = ".* has bad references." - with self.assertRaisesRegex(SystemExit, err_msg): + with six.assertRaisesRegex(self,SystemExit, err_msg): self.xml_to_tester(""+xml1+"") def test_config_reject_cyclical_references(self): @@ -2233,12 +2234,12 @@ def test_config_reject_cyclical_references(self): xml1 = """MPI_PATH""" xml2 = """MPI_LIB_NAME""" err_msg = ".* has bad references." - with self.assertRaisesRegex(SystemExit, err_msg): + with six.assertRaisesRegex(self,SystemExit, err_msg): self.xml_to_tester(""+xml1+xml2+"") xml1 = """${MPI_PATH}""" xml2 = """${MPI_LIB_NAME}""" err_msg = ".* has bad references." - with self.assertRaisesRegex(SystemExit, err_msg): + with six.assertRaisesRegex(self,SystemExit, err_msg): self.xml_to_tester(""+xml1+xml2+"") def test_variable_insertion_with_machine_specific_setting(self): @@ -2247,14 +2248,14 @@ def test_variable_insertion_with_machine_specific_setting(self): xml2 = """MPI_PATH""".format(self.test_machine) xml3 = """MPI_LIB_NAME""" err_msg = ".* has bad references." - with self.assertRaisesRegex(SystemExit, err_msg): + with six.assertRaisesRegex(self,SystemExit, err_msg): self.xml_to_tester(xml1+xml2+xml3) xml1 = """something""" xml2 = """MPI_PATH""".format(self.test_machine) xml3 = """${MPI_LIB_NAME}""" err_msg = ".* has bad references." - with self.assertRaisesRegex(SystemExit, err_msg): + with six.assertRaisesRegex(self,SystemExit, err_msg): self.xml_to_tester(xml1+xml2+xml3) From a143b37666dcce1b07b169b741e6c9c182e95082 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 4 Oct 2017 08:18:33 -0600 Subject: [PATCH 19/41] fix parenthesis --- scripts/Tools/list_acme_tests | 4 ++-- scripts/Tools/simple_compare | 2 +- scripts/lib/CIME/SystemTests/system_tests_common.py | 2 +- scripts/lib/CIME/XML/machines.py | 13 ++++++------- scripts/lib/CIME/XML/test_reporter.py | 5 ++--- scripts/lib/CIME/XML/tests.py | 2 +- scripts/lib/CIME/bless_test_results.py | 6 +++--- scripts/lib/CIME/check_lockedfiles.py | 4 ++-- scripts/lib/CIME/compare_test_results.py | 2 +- 9 files changed, 19 insertions(+), 21 deletions(-) diff --git a/scripts/Tools/list_acme_tests b/scripts/Tools/list_acme_tests index a0ab818c13d..102cbb7d50e 100755 --- a/scripts/Tools/list_acme_tests +++ b/scripts/Tools/list_acme_tests @@ -69,9 +69,9 @@ def list_tests(thing_to_list, categories): else: expect(False, "Unrecognized thing to list '{}'".format(thing_to_list)) - print(("Tested {} for test categories: {}".format(thing_to_list, ", ".join(categories)))) + print("Tested {} for test categories: {}".format(thing_to_list, ", ".join(categories))) for item in sorted(things): - print((" ", item)) + print(" ", item) ############################################################################### def _main_func(description): diff --git a/scripts/Tools/simple_compare b/scripts/Tools/simple_compare index d005d112df1..26c438ce82c 100755 --- a/scripts/Tools/simple_compare +++ b/scripts/Tools/simple_compare @@ -67,7 +67,7 @@ def _main_func(description): expect(success, "Diff between files {} and {}:\n{}".format(gold_file, compare_file, comments)) - print(("Files {} and {} MATCH".format(gold_file, compare_file))) + print("Files {} and {} MATCH".format(gold_file, compare_file)) ############################################################################### diff --git a/scripts/lib/CIME/SystemTests/system_tests_common.py b/scripts/lib/CIME/SystemTests/system_tests_common.py index 63db522869f..30019211c26 100644 --- a/scripts/lib/CIME/SystemTests/system_tests_common.py +++ b/scripts/lib/CIME/SystemTests/system_tests_common.py @@ -348,7 +348,7 @@ def compare_env_run(self, expected=None): logging.warn(" Resetting {} for test".format(key)) f1obj.set_value(key, f2obj.get_value(key, resolved=False)) else: - print(("WARNING: Found difference in test {}: case: {} original value {}".format(key, diffs[key][0], diffs[key][1]))) + print("WARNING: Found difference in test {}: case: {} original value {}".format(key, diffs[key][0], diffs[key][1])) return False return True diff --git a/scripts/lib/CIME/XML/machines.py b/scripts/lib/CIME/XML/machines.py index d762b37ff89..4108c84e0fa 100644 --- a/scripts/lib/CIME/XML/machines.py +++ b/scripts/lib/CIME/XML/machines.py @@ -300,13 +300,13 @@ def print_values(self): max_tasks_per_node = machine.find("MAX_TASKS_PER_NODE") MAX_MPITASKS_PER_NODE = machine.find("MAX_MPITASKS_PER_NODE") - print(( " {} : {} ".format(name , desc.text))) - print(( " os ", os_.text)) - print(( " compilers ",compilers.text)) + print( " {} : {} ".format(name , desc.text)) + print( " os ", os_.text) + print( " compilers ",compilers.text) if MAX_MPITASKS_PER_NODE is not None: - print(( " pes/node ",MAX_MPITASKS_PER_NODE.text)) + print(" pes/node ",MAX_MPITASKS_PER_NODE.text) if max_tasks_per_node is not None: - print(( " max_tasks/node ",max_tasks_per_node.text)) + print(" max_tasks/node ",max_tasks_per_node.text) def return_all_values(self): # return a dictionary of machines @@ -327,11 +327,10 @@ def return_all_values(self): if max_tasks_per_node is not None: max_tasks_pn = max_tasks_per_node.text - mach_dict[name] = { 'description' : desc.text, + mach_dict[name] = { 'description' : desc.text, 'os' : os_.text, 'compilers' : compilers.text, 'pes/node' : ppn, 'max_tasks/node' : max_tasks_pn } return mach_dict - diff --git a/scripts/lib/CIME/XML/test_reporter.py b/scripts/lib/CIME/XML/test_reporter.py index 560267af005..35257498b69 100644 --- a/scripts/lib/CIME/XML/test_reporter.py +++ b/scripts/lib/CIME/XML/test_reporter.py @@ -105,6 +105,5 @@ def push2testdb(self): # # Print any messages from the post command # - print((f.read())) - print((f.code)) - + print(f.read()) + print(f.code) diff --git a/scripts/lib/CIME/XML/tests.py b/scripts/lib/CIME/XML/tests.py index 5c5351543d2..944f6934687 100644 --- a/scripts/lib/CIME/XML/tests.py +++ b/scripts/lib/CIME/XML/tests.py @@ -50,4 +50,4 @@ def print_values(self, skip_infrastructure_tests=True): continue name = one_test.get("NAME") desc = one_test.find("DESC").text - print(("{}: {}".format(name, desc))) + print("{}: {}".format(name, desc)) diff --git a/scripts/lib/CIME/bless_test_results.py b/scripts/lib/CIME/bless_test_results.py index d6a1ace19d9..0b7609d89c7 100644 --- a/scripts/lib/CIME/bless_test_results.py +++ b/scripts/lib/CIME/bless_test_results.py @@ -15,7 +15,7 @@ def bless_namelists(test_name, report_only, force, baseline_name, baseline_root) # re-run create_test. # Update namelist files - print(("Test '{}' had namelist diff".format(test_name))) + print("Test '{}' had namelist diff".format(test_name)) if (not report_only and (force or input("Update namelists (y/n)? ").upper() in ["Y", "YES"])): create_test_gen_args = " -g {} ".format(baseline_name if get_model() == "cesm" else " -g -b {} ".format(baseline_name)) @@ -97,11 +97,11 @@ def bless_test_results(baseline_name, baseline_root, test_root, compiler, test_i # Now, do the bless if not nl_bless and not hist_bless: - print(("Nothing to bless for test:", test_name, " overall status:", overall_result)) + print("Nothing to bless for test:", test_name, " overall status:", overall_result) else: print("###############################################################################") - print(("Blessing results for test:", test_name, "most recent result:", overall_result)) + print("Blessing results for test:", test_name, "most recent result:", overall_result) print("###############################################################################") if not force: time.sleep(2) diff --git a/scripts/lib/CIME/check_lockedfiles.py b/scripts/lib/CIME/check_lockedfiles.py index b4c582a2493..c464878f397 100644 --- a/scripts/lib/CIME/check_lockedfiles.py +++ b/scripts/lib/CIME/check_lockedfiles.py @@ -113,8 +113,8 @@ def check_lockedfiles(case): if diffs: logging.warn("File {} has been modified".format(lfile)) for key in list(diffs.keys()): - print((" found difference in {} : case {} locked {}" - .format(key, repr(diffs[key][0]), repr(diffs[key][1])))) + print(" found difference in {} : case {} locked {}" + .format(key, repr(diffs[key][0]), repr(diffs[key][1]))) if objname == "env_mach_pes": expect(False, "Invoke case.setup --reset ") diff --git a/scripts/lib/CIME/compare_test_results.py b/scripts/lib/CIME/compare_test_results.py index 4964cff96a4..4d486807060 100644 --- a/scripts/lib/CIME/compare_test_results.py +++ b/scripts/lib/CIME/compare_test_results.py @@ -152,7 +152,7 @@ def compare_test_results(baseline_name, baseline_root, test_root, compiler, test brief_result += " {}".format(compare_comment) brief_result += "\n" - print((brief_result,)) + print(brief_result) append_status_cprnc_log(brief_result, logfile_name, test_dir) From 9a434af09b70c6b34889d54d14620aed4b8e642c Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 4 Oct 2017 08:54:58 -0600 Subject: [PATCH 20/41] avoid pylint testing six.py --- scripts/lib/CIME/code_checker.py | 2 ++ scripts/tests/scripts_regression_tests.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/lib/CIME/code_checker.py b/scripts/lib/CIME/code_checker.py index 237314f1f3b..d8b5ea9b3aa 100644 --- a/scripts/lib/CIME/code_checker.py +++ b/scripts/lib/CIME/code_checker.py @@ -58,6 +58,8 @@ def _should_pylint_skip(filepath): for dir_to_skip in list_of_directories_to_ignore: if dir_to_skip + "/" in filepath: return True + if filepath == "scripts/lib/six.py": + return True return False diff --git a/scripts/tests/scripts_regression_tests.py b/scripts/tests/scripts_regression_tests.py index 2670508c46b..44a97258225 100755 --- a/scripts/tests/scripts_regression_tests.py +++ b/scripts/tests/scripts_regression_tests.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -import io, glob, os, re, shutil, signal, sys, tempfile, \ +import glob, os, re, shutil, signal, sys, tempfile, \ threading, time, logging, unittest, getpass from xml.etree.ElementTree import ParseError From 01c0b8722c329713fac604093c7fdeafe88df667 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 4 Oct 2017 14:16:58 -0600 Subject: [PATCH 21/41] clean up list statements --- scripts/Tools/cimeteststatus | 9 ++++----- scripts/Tools/xmlquery | 2 +- .../lib/CIME/BuildTools/macroconditiontree.py | 4 ++-- .../CIME/SystemTests/system_tests_common.py | 2 +- scripts/lib/CIME/XML/compilerblock.py | 6 +++--- scripts/lib/CIME/XML/compilers.py | 5 +++-- scripts/lib/CIME/XML/entry_id.py | 4 ++-- scripts/lib/CIME/XML/env_batch.py | 4 ++-- scripts/lib/CIME/XML/generic_xml.py | 3 ++- scripts/lib/CIME/XML/grids.py | 15 +++++++-------- scripts/lib/CIME/XML/testlist.py | 5 ++--- scripts/lib/CIME/build.py | 2 +- scripts/lib/CIME/case.py | 14 +++++++------- scripts/lib/CIME/check_lockedfiles.py | 4 ++-- scripts/lib/CIME/get_timing.py | 8 ++++---- scripts/lib/CIME/hist_utils.py | 2 +- scripts/lib/CIME/namelist.py | 11 ++++++----- scripts/lib/CIME/test_scheduler.py | 4 ++-- scripts/lib/CIME/test_status.py | 2 +- scripts/lib/CIME/utils.py | 4 ++-- scripts/lib/update_acme_tests.py | 19 +++++++++---------- 21 files changed, 64 insertions(+), 65 deletions(-) diff --git a/scripts/Tools/cimeteststatus b/scripts/Tools/cimeteststatus index ce940161d42..4978d6e8ff1 100755 --- a/scripts/Tools/cimeteststatus +++ b/scripts/Tools/cimeteststatus @@ -220,11 +220,11 @@ class CimeTestStatus(): root = etree.Element('test') root.set('name', self._test_status['name']) root.set('status', self._test_status['status']) - for name, section in list(self._test_status.items()): + for name, section in self._test_status.items(): if isinstance(section, dict): elem = etree.Element('section') elem.set('name', name) - for key, value in list(section.items()): + for key, value in section.items(): child = etree.Element("{0}".format('key')) child.set('name', key) child.text = value @@ -353,10 +353,10 @@ def getSuiteInfo(specfile): for t in root.findall('test'): testlist.append(t.get('case')) - if 'machine' not in list(suiteinfo.keys()): + if 'machine' not in suiteinfo: machnodelist = t.findall('mach') suiteinfo['machine'] = machnodelist[0].text - if 'compiler' not in list(suiteinfo.keys()): + if 'compiler' not in suiteinfo: compnodelist = t.findall('compiler') suiteinfo['compiler'] = compnodelist[0].text @@ -596,4 +596,3 @@ def _main_func(): if __name__ == "__main__": _main_func() - diff --git a/scripts/Tools/xmlquery b/scripts/Tools/xmlquery index e0ccf61b812..05a7b8bc3e0 100755 --- a/scripts/Tools/xmlquery +++ b/scripts/Tools/xmlquery @@ -327,7 +327,7 @@ def _main_func(): wrapper.fix_sentence_endings = True for group in sorted(iter(results)): - if (len(variables) > 1 or len(list(results.keys())) > 1 or full) and not get_group: + if (len(variables) > 1 or len(results) > 1 or full) and not get_group: print("\nResults in group %s"%group) for var in variables: if var in results[group]: diff --git a/scripts/lib/CIME/BuildTools/macroconditiontree.py b/scripts/lib/CIME/BuildTools/macroconditiontree.py index e04721d48b3..4172f2621f5 100644 --- a/scripts/lib/CIME/BuildTools/macroconditiontree.py +++ b/scripts/lib/CIME/BuildTools/macroconditiontree.py @@ -32,7 +32,7 @@ def __init__(self, name, settings): # To make merging more effective, sort the conditions. all_conditions = [] for setting in settings: - all_conditions += list(setting.conditions.keys()) + all_conditions += setting.conditions.keys() if all_conditions: condition = sorted(all_conditions)[0] if condition is None: @@ -119,7 +119,7 @@ def merge(self, other): return other.merge(self) # If neither is a leaf and their conditions match, merge # their sets of branches. - for (cond_val, other_branch) in list(other._branches.items()): + for (cond_val, other_branch) in other._branches.items(): if cond_val in self._branches: self._branches[cond_val] = \ self._branches[cond_val].merge(other_branch) diff --git a/scripts/lib/CIME/SystemTests/system_tests_common.py b/scripts/lib/CIME/SystemTests/system_tests_common.py index 30019211c26..9ed538ec3fa 100644 --- a/scripts/lib/CIME/SystemTests/system_tests_common.py +++ b/scripts/lib/CIME/SystemTests/system_tests_common.py @@ -343,7 +343,7 @@ def compare_env_run(self, expected=None): f1obj = EnvRun(self._caseroot, "env_run.xml", components=components) f2obj = EnvRun(self._caseroot, os.path.join(LOCKED_DIR, "env_run.orig.xml"), components=components) diffs = f1obj.compare_xml(f2obj) - for key in list(diffs.keys()): + for key in diffs.keys(): if expected is not None and key in expected: logging.warn(" Resetting {} for test".format(key)) f1obj.set_value(key, f2obj.get_value(key, resolved=False)) diff --git a/scripts/lib/CIME/XML/compilerblock.py b/scripts/lib/CIME/XML/compilerblock.py index 873ed398045..a975c7efb27 100644 --- a/scripts/lib/CIME/XML/compilerblock.py +++ b/scripts/lib/CIME/XML/compilerblock.py @@ -225,7 +225,7 @@ def _add_elem_to_lists(self, name, elem, value_lists): of all settings for each variable. """ # Skip this if the element's MPILIB is not valid. - if "MPILIB" in list(elem.keys()) and \ + if "MPILIB" in elem.keys() and \ not self._machobj.is_valid_MPIlib(elem.get("MPILIB")): return setting, depends = self._elem_to_setting(elem) @@ -259,13 +259,13 @@ def matches_machine(self): before add_settings_to_lists if machine-specific output is needed. """ self._specificity = 0 - if "MACH" in list(self._compiler_elem.keys()): + if "MACH" in self._compiler_elem.keys(): if self._machobj.get_machine_name() == \ self._compiler_elem.get("MACH"): self._specificity += 2 else: return False - if "OS" in list(self._compiler_elem.keys()): + if "OS" in self._compiler_elem.keys(): if self._machobj.get_value("OS") == self._compiler_elem.get("OS"): self._specificity += 1 else: diff --git a/scripts/lib/CIME/XML/compilers.py b/scripts/lib/CIME/XML/compilers.py index 6ed1c5feef6..1bbcf93d341 100644 --- a/scripts/lib/CIME/XML/compilers.py +++ b/scripts/lib/CIME/XML/compilers.py @@ -10,6 +10,7 @@ from CIME.BuildTools.makemacroswriter import MakeMacroWriter from CIME.BuildTools.cmakemacroswriter import CMakeMacroWriter from CIME.BuildTools.macroconditiontree import merge_optional_trees +import six logger = logging.getLogger(__name__) @@ -160,7 +161,7 @@ def write_macros_file(self, macros_file="Macros.make", output_format="make", xml else: format_ = output_format - if isinstance(macros_file, str): + if isinstance(macros_file, six.string_types): with open(macros_file, "w") as macros: self._write_macros_file_v2(format_, macros) else: @@ -205,7 +206,7 @@ def _write_macros_file_v2(self, build_system, output, xml=None): while value_lists: # Variables that are ready to be written. ready_variables = [ - var_name for var_name in list(value_lists.keys()) + var_name for var_name in value_lists if value_lists[var_name].depends <= vars_written ] expect(len(ready_variables) > 0, diff --git a/scripts/lib/CIME/XML/entry_id.py b/scripts/lib/CIME/XML/entry_id.py index 23cf26f7bb6..94b2aca8423 100644 --- a/scripts/lib/CIME/XML/entry_id.py +++ b/scripts/lib/CIME/XML/entry_id.py @@ -83,7 +83,7 @@ def _get_value_match(self, node, attributes=None, exact_match=False): # For each node in the list start a score. score = 0 if attributes: - for attribute in list(vnode.keys()): + for attribute in vnode.keys(): # For each attribute, add to the score. score += 1 # If some attribute is specified that we don't know about, @@ -360,7 +360,7 @@ def add_elements_by_group(self, srcobj, attributes=None, infile=None): gname = "group_not_set" # If group with id=$gname does not exist in self.groups # then create the group node and add it to infile file - if gname not in list(self.groups.keys()): + if gname not in self.groups.keys(): newgroup = ET.Element("group") newgroup.set("id",gname) # initialize an empty list diff --git a/scripts/lib/CIME/XML/env_batch.py b/scripts/lib/CIME/XML/env_batch.py index b0f585f289e..ef407455e88 100644 --- a/scripts/lib/CIME/XML/env_batch.py +++ b/scripts/lib/CIME/XML/env_batch.py @@ -123,7 +123,7 @@ def create_job_groups(self, batch_jobs): for name, jdict in batch_jobs: new_job_group = ET.Element("group") new_job_group.set("id", name) - for field in list(jdict.keys()): + for field in jdict.keys(): val = jdict[field] node = ET.SubElement(new_job_group, "entry", {"id":field,"value":val}) tnode = ET.SubElement(node, "type") @@ -359,7 +359,7 @@ def submit_jobs(self, case, no_batch=False, job=None, skip_pnl=False, if self._prereq_jobid is not None: jobid = self._prereq_jobid for dep in deps: - if dep in list(depid.keys()) and depid[dep] is not None: + if dep in depid: jobid += " " + str(depid[dep]) #TODO: doubt these will be used # elif dep == "and": diff --git a/scripts/lib/CIME/XML/generic_xml.py b/scripts/lib/CIME/XML/generic_xml.py index fc02c2459a8..836298af0b9 100644 --- a/scripts/lib/CIME/XML/generic_xml.py +++ b/scripts/lib/CIME/XML/generic_xml.py @@ -6,6 +6,7 @@ from distutils.spawn import find_executable from xml.dom import minidom import getpass +import six logger = logging.getLogger(__name__) @@ -125,7 +126,7 @@ def get_nodes(self, nodename, attributes=None, root=None, xpath=None): for key, value in attributes.items(): if value is not None: - expect(isinstance(value, str), + expect(isinstance(value, six.string_types), " Bad value passed for key {}".format(key)) xpath = ".//{}[@{}=\'{}\']".format(nodename, key, value) logger.debug("xpath is {}".format(xpath)) diff --git a/scripts/lib/CIME/XML/grids.py b/scripts/lib/CIME/XML/grids.py index b443ef06c37..6d0a208ac3a 100644 --- a/scripts/lib/CIME/XML/grids.py +++ b/scripts/lib/CIME/XML/grids.py @@ -533,7 +533,7 @@ def _print_values_v1(self, long_output=None): logger.info(" short name: {}".format(sname)) if alias is not None: logger.info(" alias: {}".format(alias)) - for attr, value in list(grid_node.items()): + for attr, value in grid_node.items(): if attr == 'compset': logger.info(" compset match: {}".format(value)) @@ -582,7 +582,7 @@ def _get_all_values_v1(self): "alias" : alias, } compset_list = list() - for attr, value in list(grid_node.items()): + for attr, value in grid_node.items(): if attr == 'compset': compset_list.append(value) @@ -601,7 +601,7 @@ def _get_all_values_v1(self): domain_list.append({'domain':child.tag, 'text':child.text}) grid_info.update({'domains': domain_list}) - + # add mapping files grids = [ ("atm_grid", component_grids[0]), ("lnd_grid", component_grids[1]), ("ocn_grid", component_grids[2]), \ ("rof_grid", component_grids[3]), ("glc_grid", component_grids[5]), ("wav_grid", component_grids[6]) ] @@ -682,16 +682,16 @@ def _get_all_values_v2(self): gridnames = [] for grid_node in grid_nodes: gridnames.append(grid_node.text) - grids += grid_node.get("name") + ":" + grid_node.text + " " + grids += grid_node.get("name") + ":" + grid_node.text + " " grids = " non-default grids are: %s" %grids mask = "" mask_nodes = self.get_nodes("mask", root=model_grid_node) for mask_node in mask_nodes: mask += "\n mask is: %s" %(mask_node.text) - - grids_dict[alias] = {'aliases':aliases, - 'grids':grids, + + grids_dict[alias] = {'aliases':aliases, + 'grids':grids, 'mask':mask } gridnames = set(gridnames) @@ -712,4 +712,3 @@ def return_all_values(self): (default_comp_grids, all_grids) = self._get_all_values_v2() return help_text, default_comp_grids, all_grids - diff --git a/scripts/lib/CIME/XML/testlist.py b/scripts/lib/CIME/XML/testlist.py index bc3e4f7a49b..38a49650fa2 100644 --- a/scripts/lib/CIME/XML/testlist.py +++ b/scripts/lib/CIME/XML/testlist.py @@ -115,7 +115,7 @@ def _get_testsv2(self, machine=None, category=None, compiler=None, compset=None, machnodes = self.get_nodes("machine",machatts,root=tnode) if machnodes: this_test_node = {} - for key, value in list(tnode.items()): + for key, value in tnode.items(): if key == "name": this_test_node["testname"] = value else: @@ -124,7 +124,7 @@ def _get_testsv2(self, machine=None, category=None, compiler=None, compset=None, for mach in machnodes: # this_test_node can include multiple tests this_test = dict(this_test_node) - for key, value in list(mach.items()): + for key, value in mach.items(): if key == "name": this_test["machine"] = value else: @@ -157,4 +157,3 @@ def get_tests(self, machine=None, category=None, compiler=None, compset=None, gr else: logger.critical("Did not recognize testlist file version {} for file {}" .format(self.get_version(), self.filename)) - diff --git a/scripts/lib/CIME/build.py b/scripts/lib/CIME/build.py index 3d9857e21fe..52c1e5bcc05 100644 --- a/scripts/lib/CIME/build.py +++ b/scripts/lib/CIME/build.py @@ -387,7 +387,7 @@ def _case_build_impl(caseroot, case, sharedlib_only, model_only, buildlist): # Retrieve relevant case data # This environment variable gets set for cesm Make and # needs to be unset before building again. - if "MODEL" in list(os.environ.keys()): + if "MODEL" in os.environ: del os.environ["MODEL"] build_threaded = case.get_build_threaded() casetools = case.get_value("CASETOOLS") diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index 25c6261f132..e481b8aea59 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -403,7 +403,7 @@ def set_valid_values(self, item, valid_values): return result def set_lookup_value(self, item, value): - if item in list(self.lookups.keys()) and self.lookups[item] is not None: + if item in self.lookups and self.lookups[item] is not None: logger.warning("Item {} already in lookups with value {}".format(item,self.lookups[item])) else: logger.debug("Setting in lookups: item {}, value {}".format(item,value)) @@ -411,7 +411,7 @@ def set_lookup_value(self, item, value): def clean_up_lookups(self, allow_undefined=False): # put anything in the lookups table into existing env objects - for key,value in list(self.lookups.items()): + for key,value in self.lookups.items(): logger.debug("lookup key {} value {}".format(key, value)) result = self.set_value(key,value, allow_undefined=allow_undefined) if result is not None: @@ -695,7 +695,7 @@ def _setup_mach_pes(self, pecount, multi_driver, ninst, machine_name, mpilib): mach_pes_obj = self.get_env("mach_pes") if other is not None: - for key, value in list(other.items()): + for key, value in other.items(): self.set_value(key, value) totaltasks = [] @@ -734,10 +734,10 @@ def _setup_mach_pes(self, pecount, multi_driver, ninst, machine_name, mpilib): mach_pes_obj.set_value(key, ninst) key = "NTASKS_{}".format(compclass) - if key not in list(pes_ntasks.keys()): + if key not in pes_ntasks: mach_pes_obj.set_value(key,1) key = "NTHRDS_{}".format(compclass) - if compclass not in list(pes_nthrds.keys()): + if compclass not in pes_nthrds: mach_pes_obj.set_value(compclass,1) return pesize @@ -769,7 +769,7 @@ def configure(self, compset_name, grid_name, machine_name=None, gridinfo = grids.get_grid_info(name=grid_name, compset=self._compsetname) self._gridname = gridinfo["GRID"] - for key,value in list(gridinfo.items()): + for key,value in gridinfo.items(): logger.debug("Set grid {} {}".format(key,value)) self.set_lookup_value(key,value) @@ -968,7 +968,7 @@ def _set_pio_xml(self): mpilib = self.get_value("MPILIB") defaults = pioobj.get_defaults(grid=grid,compset=compset,mach=mach,compiler=compiler, mpilib=mpilib) - for vid, value in list(defaults.items()): + for vid, value in defaults.items(): self.set_value(vid,value) def _create_caseroot_tools(self): diff --git a/scripts/lib/CIME/check_lockedfiles.py b/scripts/lib/CIME/check_lockedfiles.py index c464878f397..1e9d87a7aae 100644 --- a/scripts/lib/CIME/check_lockedfiles.py +++ b/scripts/lib/CIME/check_lockedfiles.py @@ -112,7 +112,7 @@ def check_lockedfiles(case): diffs = f1obj.compare_xml(f2obj) if diffs: logging.warn("File {} has been modified".format(lfile)) - for key in list(diffs.keys()): + for key in diffs.keys(): print(" found difference in {} : case {} locked {}" .format(key, repr(diffs[key][0]), repr(diffs[key][1]))) @@ -124,7 +124,7 @@ def check_lockedfiles(case): elif objname == "env_build": logging.warn("Setting build complete to False") case.set_value("BUILD_COMPLETE", False) - if "PIO_VERSION" in list(diffs.keys()): + if "PIO_VERSION" in diffs: case.set_value("BUILD_STATUS", 2) logging.critical("Changing PIO_VERSION requires running " "case.build --clean-all and rebuilding") diff --git a/scripts/lib/CIME/get_timing.py b/scripts/lib/CIME/get_timing.py index 38b6153b353..4353f78a860 100644 --- a/scripts/lib/CIME/get_timing.py +++ b/scripts/lib/CIME/get_timing.py @@ -145,7 +145,7 @@ def _getTiming(self, inst=0): else: pecost = totalpes - for m in list(self.models.values()): + for m in self.models.values(): for key in ["NTASKS", "ROOTPE", "PSTRID", "NTHRDS", "NINST"]: if key == "NINST" and m.name == "CPL": m.ninst = 1 @@ -215,7 +215,7 @@ def _getTiming(self, inst=0): if inittype == "TRUE": odays = odays - (tlen/ocn_ncpl) - peminmax = max([m.rootpe for m in list(self.models.values())])+1 + peminmax = max([m.rootpe for m in self.models.values()])+1 if ncpl_base_period in ["decade","year","day"] and int(adays) > 0: adays = int(adays) if tlen % ocn_ncpl == 0: @@ -223,7 +223,7 @@ def _getTiming(self, inst=0): self.adays = adays maxoffset = 40 extraoff = 20 - for m in list(self.models.values()): + for m in self.models.values(): m.offset = int((maxoffset*m.rootpe)/peminmax) + extraoff cpl.offset = 0 try: @@ -293,7 +293,7 @@ def _getTiming(self, inst=0): tmax = tmax + wtmin + correction ocn.tmax += ocnrunitime - for m in list(self.models.values()): + for m in self.models.values(): m.tmaxr = 0 if m.tmax > 0: m.tmaxr = adays*86400.0/(m.tmax*365.0) diff --git a/scripts/lib/CIME/hist_utils.py b/scripts/lib/CIME/hist_utils.py index a06d5468d5e..045e913f719 100644 --- a/scripts/lib/CIME/hist_utils.py +++ b/scripts/lib/CIME/hist_utils.py @@ -44,7 +44,7 @@ def _get_latest_hist_files(testcase, model, from_dir, suffix=""): ext = get_extension(model, hist) latest_files[ext] = hist - for key in list(latest_files.keys()): + for key in latest_files.keys(): histlist.append(latest_files[key]) return histlist diff --git a/scripts/lib/CIME/namelist.py b/scripts/lib/CIME/namelist.py index 43377028bf7..5aa49166a68 100644 --- a/scripts/lib/CIME/namelist.py +++ b/scripts/lib/CIME/namelist.py @@ -108,6 +108,7 @@ from CIME.XML.standard_module_setup import * from CIME.utils import expect +import six logger = logging.getLogger(__name__) @@ -732,21 +733,21 @@ def compress_literal_list(literals): # Otherwise, write out the previous literal and start tracking the # new one. rep_str = str(num_reps) + '*' if num_reps > 1 else '' - if isinstance(old_literal, str): + if isinstance(old_literal, six.string_types): compressed.append(rep_str + old_literal) else: compressed.append(rep_str + str(old_literal)) old_literal = literal num_reps = 1 rep_str = str(num_reps) + '*' if num_reps > 1 else '' - if isinstance(old_literal, str): + if isinstance(old_literal, six.string_types): compressed.append(rep_str + old_literal) else: compressed.append(rep_str + str(old_literal)) return compressed else: for literal in literals: - if isinstance(literal, str): + if isinstance(literal, six.string_types): compressed.append(literal) else: compressed.append(str(literal)) @@ -833,7 +834,7 @@ def parse(in_file=None, text=None, groupless=False, convert_tab_to_space=True): "Must specify an input file or text to the namelist parser.") expect(in_file is None or text is None, "Cannot specify both input file and text to the namelist parser.") - if isinstance(in_file, str) or isinstance(in_file, str): + if isinstance(in_file, six.string_types): logger.debug("Reading namelist at: {}".format(in_file)) with open(in_file) as in_file_obj: text = in_file_obj.read() @@ -1134,7 +1135,7 @@ def write(self, out_file, groups=None, append=False, format_='nml', sorted_group """ expect(format_ in ('nml', 'rc', 'nmlcontents'), "Namelist.write: unexpected output format {!r}".format(str(format_))) - if isinstance(out_file, str) or isinstance(out_file, str): + if isinstance(out_file, six.string_types): logger.debug("Writing namelist to: {}".format(out_file)) flag = 'a' if append else 'w' with open(out_file, flag) as file_obj: diff --git a/scripts/lib/CIME/test_scheduler.py b/scripts/lib/CIME/test_scheduler.py index 98efb015fe4..c3f7aa87e6e 100644 --- a/scripts/lib/CIME/test_scheduler.py +++ b/scripts/lib/CIME/test_scheduler.py @@ -643,7 +643,7 @@ def _get_procs_needed(self, test, phase, threads_in_flight=None, no_batch=False) # Will force serialization of sharedlib builds # TODO - instead of serializing, compute all library configs needed and build # them all in parallel - for _, _, running_phase in list(threads_in_flight.values()): + for _, _, running_phase in threads_in_flight.values(): if (running_phase == SHAREDLIB_BUILD_PHASE): return self._proc_pool + 1 @@ -774,7 +774,7 @@ def _producer(self): # No free resources, wait for something in flight to finish self._wait_for_something_to_finish(threads_in_flight) - for unfinished_thread, _, _ in list(threads_in_flight.values()): + for unfinished_thread, _, _ in threads_in_flight.values(): unfinished_thread.join() ########################################################################### diff --git a/scripts/lib/CIME/test_status.py b/scripts/lib/CIME/test_status.py index 87cb239bc1a..9bbb4a9c397 100644 --- a/scripts/lib/CIME/test_status.py +++ b/scripts/lib/CIME/test_status.py @@ -192,7 +192,7 @@ def set_status(self, phase, status, comments=""): if subsequent_phase in self._phase_statuses: del self._phase_statuses[subsequent_phase] if subsequent_phase.startswith(COMPARE_PHASE): - for stored_phase in list(self._phase_statuses.keys()): + for stored_phase in self._phase_statuses.keys(): if stored_phase.startswith(COMPARE_PHASE): del self._phase_statuses[stored_phase] diff --git a/scripts/lib/CIME/utils.py b/scripts/lib/CIME/utils.py index 98a02b4bc51..569eb9ce0b1 100644 --- a/scripts/lib/CIME/utils.py +++ b/scripts/lib/CIME/utils.py @@ -249,12 +249,12 @@ def run_cmd(cmd, input_str=None, from_dir=None, verbose=None, # Real defaults for these value should be subprocess.PIPE if arg_stdout is _hack: arg_stdout = subprocess.PIPE - elif isinstance(arg_stdout, str): + elif isinstance(arg_stdout, six.string_types): arg_stdout = _convert_to_fd(arg_stdout, from_dir) if arg_stderr is _hack: arg_stderr = subprocess.STDOUT if combine_output else subprocess.PIPE - elif isinstance(arg_stderr, str): + elif isinstance(arg_stderr, six.string_types): arg_stderr = _convert_to_fd(arg_stdout, from_dir) if (verbose != False and (verbose or logger.isEnabledFor(logging.DEBUG))): diff --git a/scripts/lib/update_acme_tests.py b/scripts/lib/update_acme_tests.py index 87a903a52ac..c4419540e80 100644 --- a/scripts/lib/update_acme_tests.py +++ b/scripts/lib/update_acme_tests.py @@ -1,6 +1,7 @@ import CIME.utils from CIME.utils import expect, convert_to_seconds, parse_test_name from CIME.XML.machines import Machines +import six # Here are the tests belonging to acme suites. Format is # ... @@ -183,27 +184,27 @@ def get_test_suite(suite, machine=None, compiler=None): tests = [] for item in tests_raw: test_mod = None - if (isinstance(item, str)): + if (isinstance(item, six.string_types)): test_name = item else: expect(isinstance(item, tuple), "Bad item type for item '{}'".format(str(item))) expect(len(item) in [2, 3], "Expected two or three items in item '{}'".format(str(item))) - expect(isinstance(item[0], str), "Expected string in first field of item '{}'".format(str(item))) - expect(isinstance(item[1], str), "Expected string in second field of item '{}'".format(str(item))) + expect(isinstance(item[0], six.string_types), "Expected string in first field of item '{}'".format(str(item))) + expect(isinstance(item[1], six.string_types), "Expected string in second field of item '{}'".format(str(item))) test_name = item[0] if (len(item) == 2): test_mod = item[1] else: - expect(type(item[2]) in [str, tuple], "Expected string or tuple for third field of item '{}'".format(str(item))) - test_mod_machines = [item[2]] if isinstance(item[2], str) else item[2] + expect(type(item[2]) in [six.string_types, tuple], "Expected string or tuple for third field of item '{}'".format(str(item))) + test_mod_machines = [item[2]] if isinstance(item[2], six.string_types) else item[2] if (machine in test_mod_machines): test_mod = item[1] tests.append(CIME.utils.get_full_test_name(test_name, machine=machine, compiler=compiler, testmod=test_mod)) if (inherits_from is not None): - inherits_from = [inherits_from] if isinstance(inherits_from, str) else inherits_from + inherits_from = [inherits_from] if isinstance(inherits_from, six.string_types) else inherits_from for inherits in inherits_from: inherited_tests = get_test_suite(inherits, machine, compiler) @@ -322,14 +323,14 @@ def get_recommended_test_time(test_full_name): _, rec_time, tests_raw = _TEST_SUITES[suite] for item in tests_raw: test_mod = None - if (isinstance(item, str)): + if (isinstance(item, six.string_types)): test_name = item else: test_name = item[0] if (len(item) == 2): test_mod = item[1] else: - test_mod_machines = [item[2]] if isinstance(item[2], str) else item[2] + test_mod_machines = [item[2]] if isinstance(item[2], six.string_types) else item[2] if (machine in test_mod_machines): test_mod = item[1] @@ -365,5 +366,3 @@ def sort_by_time(test_one, test_two): a = convert_to_seconds(rec2) b = convert_to_seconds(rec1) return (a < b) - (b < a) - - From a08581d85ae00237f31b7c7ed89e178c89dac7e0 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 4 Oct 2017 16:09:36 -0600 Subject: [PATCH 22/41] edited too much --- scripts/lib/CIME/XML/env_batch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/lib/CIME/XML/env_batch.py b/scripts/lib/CIME/XML/env_batch.py index ef407455e88..6e150132f70 100644 --- a/scripts/lib/CIME/XML/env_batch.py +++ b/scripts/lib/CIME/XML/env_batch.py @@ -359,7 +359,7 @@ def submit_jobs(self, case, no_batch=False, job=None, skip_pnl=False, if self._prereq_jobid is not None: jobid = self._prereq_jobid for dep in deps: - if dep in depid: + if dep in depid and depid[dep] is not None: jobid += " " + str(depid[dep]) #TODO: doubt these will be used # elif dep == "and": From e890236489e4df28514ae2adb677743a58495d52 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 5 Oct 2017 06:02:32 -0600 Subject: [PATCH 23/41] hard code list for python3 --- scripts/tests/list_tests | 66 +++++++++++++++++++++++++++++----------- 1 file changed, 49 insertions(+), 17 deletions(-) diff --git a/scripts/tests/list_tests b/scripts/tests/list_tests index 4f869e7c143..158776cd2b7 100755 --- a/scripts/tests/list_tests +++ b/scripts/tests/list_tests @@ -1,26 +1,58 @@ #!/usr/bin/env python - +# This script will print the list of test classes in +# scripts_regression_tests.py, however discover doesnt seem to work +# correctly in python3 so the list is hard coded. +# When run in python2 the hard coded list is checked for correctness. +# import unittest +import sys DEBUG = False #pylint: disable=protected-access def list_tests_from(): - loader = unittest.TestLoader() - suite = loader.discover(".", pattern="scripts_regression_tests.py") - test_classes = [] - for alltests in suite: - tests = alltests._tests - if len(tests): - for atest in tests: - if DEBUG: - print(atest) - for btest in atest._tests: - btestname = btest.__str__().split() - test_classes.append(btestname[1][1:-1].split('.')[1]) - # add this explicitly, not captured by the above - test_classes.append("B_CheckCode") - for ctest in sorted(list(set(test_classes))): - print(ctest) + test_classes_hard = hard_coded_list() + if sys.hexversion < 0x03000000: + loader = unittest.TestLoader() + suite = loader.discover(".", pattern="scripts_regression_tests.py") + test_classes = [] + for alltests in suite: + tests = alltests._tests + if len(tests): + for atest in tests: + if DEBUG: + print(atest) + for btest in atest._tests: + btestname = btest.__str__().split() + test_classes.append(btestname[1][1:-1].split('.')[1]) + # add this explicitly, not captured by the above + test_classes.append("B_CheckCode") + if set(test_classes) != set(test_classes_hard): + print ("Error: hard coded test_class list out of sync with actual, please fix") + test_classes_hard = test_classes + + for ctest in sorted(list(set(test_classes_hard))): + print(ctest) + +def hard_coded_list(): + return ('A_RunUnitTests', + 'B_CheckCode', + 'G_TestMacrosBasic', + 'H_TestMakeMacros', + 'I_TestCMakeMacros', + 'J_TestCreateNewcase', + 'K_TestCimeCase', + 'L_TestSaveTimings', + 'M_TestWaitForTests', + 'N_TestUnitTest', + 'O_TestTestScheduler', + 'P_TestJenkinsGenericJob', + 'Q_TestBlessTestResults', + 'S_TestManageAndQuery', + 'T_TestRunRestart', + 'X_TestSingleSubmit', + 'Z_FullSystemTest') + + if __name__ == "__main__": # Include the directories From 57fe4c860adc8ebae4362b32b5b34b8756c30b2a Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 5 Oct 2017 06:31:29 -0600 Subject: [PATCH 24/41] exit on error --- scripts/tests/list_tests | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/tests/list_tests b/scripts/tests/list_tests index 158776cd2b7..886cd3994ee 100755 --- a/scripts/tests/list_tests +++ b/scripts/tests/list_tests @@ -28,6 +28,7 @@ def list_tests_from(): test_classes.append("B_CheckCode") if set(test_classes) != set(test_classes_hard): print ("Error: hard coded test_class list out of sync with actual, please fix") + sys.exit(1) test_classes_hard = test_classes for ctest in sorted(list(set(test_classes_hard))): From 9cf9b4deb9733d414bd9330351c9766ea276aa7f Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 5 Oct 2017 10:14:21 -0600 Subject: [PATCH 25/41] fix unittest fails for py3 --- .../CIME/SystemTests/system_tests_common.py | 7 +++---- scripts/lib/CIME/utils.py | 19 ++++++++++++------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/scripts/lib/CIME/SystemTests/system_tests_common.py b/scripts/lib/CIME/SystemTests/system_tests_common.py index 9ed538ec3fa..0fed6bc94be 100644 --- a/scripts/lib/CIME/SystemTests/system_tests_common.py +++ b/scripts/lib/CIME/SystemTests/system_tests_common.py @@ -155,15 +155,14 @@ def run(self): self._check_for_memleak() - except: + except BaseException as msg: success = False - msg = sys.exc_info()[1].message - if "RUN FAIL" in msg: + if "RUN FAIL" in str(msg): # Don't want to print stacktrace for a model failure since that # is not a CIME/infrastructure problem. excmsg = msg else: - excmsg = "Exception during run:\n{}\n{}".format(sys.exc_info()[1], traceback.format_exc()) + excmsg = "Exception during run:\n{}\n{}".format(msg, traceback.format_exc()) logger.warning(excmsg) append_testlog(excmsg) diff --git a/scripts/lib/CIME/utils.py b/scripts/lib/CIME/utils.py index 569eb9ce0b1..99d7eb8cc3d 100644 --- a/scripts/lib/CIME/utils.py +++ b/scripts/lib/CIME/utils.py @@ -2,7 +2,7 @@ Common functions used by cime python scripts Warning: you cannot use CIME Classes in this module as it causes circular dependencies """ -import logging, gzip, sys, os, time, re, shutil, glob, string, random, imp, errno, signal +import io, logging, gzip, sys, os, time, re, shutil, glob, string, random, imp, errno, signal import stat as statlib import warnings import six @@ -277,12 +277,17 @@ def run_cmd(cmd, input_str=None, from_dir=None, verbose=None, output = output.strip() if output is not None else output errput = errput.strip() if errput is not None else errput stat = proc.wait() - - if isinstance(arg_stdout, file): - arg_stdout.close() # pylint: disable=no-member - - if isinstance(arg_stderr, file) and arg_stderr is not arg_stdout: - arg_stderr.close() # pylint: disable=no-member + if six.PY2: + if isinstance(arg_stdout, file): + arg_stdout.close() # pylint: disable=no-member + if isinstance(arg_stderr, file) and arg_stderr is not arg_stdout: + arg_stderr.close() # pylint: disable=no-member + else: + if isinstance(arg_stdout, io.IOBase): + arg_stdout.close() # pylint: disable=no-member + if isinstance(arg_stderr, io.IOBase) and arg_stderr is not arg_stdout: + arg_stderr.close() # pylint: disable=no-member + if (verbose != False and (verbose or logger.isEnabledFor(logging.DEBUG))): if stat != 0: From 283f8e80cd4a2e66bed2743a614c36cb34d41954 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 5 Oct 2017 12:40:26 -0600 Subject: [PATCH 26/41] more py3 fixes --- scripts/lib/CIME/code_checker.py | 4 ++-- scripts/tests/scripts_regression_tests.py | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/lib/CIME/code_checker.py b/scripts/lib/CIME/code_checker.py index d8b5ea9b3aa..9ce8102d45f 100644 --- a/scripts/lib/CIME/code_checker.py +++ b/scripts/lib/CIME/code_checker.py @@ -69,8 +69,8 @@ def get_all_checkable_files(): cimeroot = get_cime_root() all_git_files = run_cmd_no_fail("git ls-files", from_dir=cimeroot, verbose=False).splitlines() - files_to_test = [item for item in all_git_files - if ((item.endswith(".py") or is_python_executable(os.path.join(cimeroot, item))) and not _should_pylint_skip(item))] + files_to_test = [str(item) for item in all_git_files + if ((str(item).endswith(".py") or is_python_executable(os.path.join(cimeroot, str(item)))) and not _should_pylint_skip(str(item)))] return files_to_test ############################################################################### diff --git a/scripts/tests/scripts_regression_tests.py b/scripts/tests/scripts_regression_tests.py index 44a97258225..0e8fa3a5eb9 100755 --- a/scripts/tests/scripts_regression_tests.py +++ b/scripts/tests/scripts_regression_tests.py @@ -4,13 +4,13 @@ threading, time, logging, unittest, getpass from xml.etree.ElementTree import ParseError -import six LIB_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)),"..","lib") sys.path.append(LIB_DIR) # Remove all pyc files to ensure we're testing the right things import subprocess subprocess.call('/bin/rm $(find . -name "*.pyc")', shell=True, cwd=LIB_DIR) +import six from CIME.utils import run_cmd, run_cmd_no_fail, get_lids, get_current_commit import update_acme_tests @@ -2334,7 +2334,7 @@ def check_for_pylint(): pylint = find_executable("pylint") if pylint is not None: output = run_cmd_no_fail("pylint --version") - pylintver = re.search(r"pylint\s+(\d+)[.](\d+)[.](\d+)", output) + pylintver = re.search(r"pylint\s+(\d+)[.](\d+)[.](\d+)", str(output)) major = int(pylintver.group(1)) minor = int(pylintver.group(2)) if pylint is None or major < 1 or (major == 1 and minor < 5): @@ -2434,8 +2434,7 @@ def _main_func(): try: unittest.main(verbosity=2, catchbreak=True) - except SystemExit: - had_fails = sys.exc_info()[1].message + except SystemExit as had_fails: if had_fails: print("Detected failures, leaving directory:", TEST_ROOT) else: From 370a2a9c246037d382ed68b0d1e54c0e090ec635 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 5 Oct 2017 15:35:40 -0600 Subject: [PATCH 27/41] more py3 fixes --- .../CIME/SystemTests/system_tests_common.py | 2 +- scripts/lib/CIME/XML/entry_id.py | 4 +- scripts/lib/CIME/XML/env_batch.py | 4 +- scripts/lib/CIME/XML/namelist_definition.py | 4 +- scripts/lib/CIME/XML/pes.py | 10 +-- scripts/lib/CIME/build.py | 2 +- scripts/lib/CIME/case_clone.py | 4 +- scripts/lib/CIME/case_test.py | 2 +- scripts/lib/CIME/check_lockedfiles.py | 8 +-- scripts/lib/CIME/code_checker.py | 8 ++- .../test_utils/test_user_nl_utils.py | 2 +- .../lib/CIME/tests/test_user_mod_support.py | 2 +- scripts/lib/CIME/utils.py | 28 ++++++--- scripts/tests/list_tests | 62 +++++-------------- scripts/tests/scripts_regression_tests.py | 8 +-- .../buildlib.internal_components | 1 - src/build_scripts/buildlib.pio | 4 +- 17 files changed, 68 insertions(+), 87 deletions(-) diff --git a/scripts/lib/CIME/SystemTests/system_tests_common.py b/scripts/lib/CIME/SystemTests/system_tests_common.py index 0fed6bc94be..eeaa9997bab 100644 --- a/scripts/lib/CIME/SystemTests/system_tests_common.py +++ b/scripts/lib/CIME/SystemTests/system_tests_common.py @@ -344,7 +344,7 @@ def compare_env_run(self, expected=None): diffs = f1obj.compare_xml(f2obj) for key in diffs.keys(): if expected is not None and key in expected: - logging.warn(" Resetting {} for test".format(key)) + logging.warning(" Resetting {} for test".format(key)) f1obj.set_value(key, f2obj.get_value(key, resolved=False)) else: print("WARNING: Found difference in test {}: case: {} original value {}".format(key, diffs[key][0], diffs[key][1])) diff --git a/scripts/lib/CIME/XML/entry_id.py b/scripts/lib/CIME/XML/entry_id.py index 94b2aca8423..8928a52add5 100644 --- a/scripts/lib/CIME/XML/entry_id.py +++ b/scripts/lib/CIME/XML/entry_id.py @@ -39,7 +39,7 @@ def set_default_value(self, vid, val): if node is not None: val = self.set_element_text("default_value", val, root=node) if val is None: - logger.warn("Called set_default_value on a node without default_value field") + logger.warning("Called set_default_value on a node without default_value field") return val @@ -211,7 +211,7 @@ def _set_valid_values(self, node, new_valid_values): current_value = node.get("value") valid_values_list = self._get_valid_values(node) if current_value is not None and current_value not in valid_values_list: - logger.warn("WARNING: Current setting for {} not in new valid values. Updating setting to \"{}\"".format(node.get("id"), valid_values_list[0])) + logger.warning("WARNING: Current setting for {} not in new valid values. Updating setting to \"{}\"".format(node.get("id"), valid_values_list[0])) self._set_value(node, valid_values_list[0]) return new_valid_values diff --git a/scripts/lib/CIME/XML/env_batch.py b/scripts/lib/CIME/XML/env_batch.py index 6e150132f70..add5de20900 100644 --- a/scripts/lib/CIME/XML/env_batch.py +++ b/scripts/lib/CIME/XML/env_batch.py @@ -372,7 +372,7 @@ def submit_jobs(self, case, no_batch=False, job=None, skip_pnl=False, if slen == 0: jobid = None - logger.warn("job is {}".format(job)) + logger.warning("job is {}".format(job)) result = self._submit_single_job(case, job, jobid, no_batch=no_batch, skip_pnl=skip_pnl, @@ -394,7 +394,7 @@ def submit_jobs(self, case, no_batch=False, job=None, skip_pnl=False, def _submit_single_job(self, case, job, depid=None, no_batch=False, skip_pnl=False, mail_user=None, mail_type='never', batch_args=None, dry_run=False): - logger.warn("Submit job {}".format(job)) + logger.warning("Submit job {}".format(job)) batch_system = self.get_value("BATCH_SYSTEM", subgroup=None) if batch_system is None or batch_system == "none" or no_batch: # Import here to avoid circular include diff --git a/scripts/lib/CIME/XML/namelist_definition.py b/scripts/lib/CIME/XML/namelist_definition.py index 6a8a631015c..a8054d6b6e5 100644 --- a/scripts/lib/CIME/XML/namelist_definition.py +++ b/scripts/lib/CIME/XML/namelist_definition.py @@ -270,7 +270,7 @@ def is_valid_value(self, name, value): if not is_valid_fortran_namelist_literal(type_, scalar): invalid.append(scalar) if len(invalid) > 0: - logger.warn("Invalid values {}".format(invalid)) + logger.warning("Invalid values {}".format(invalid)) return False # Now that we know that the strings as input are valid Fortran, do some @@ -296,7 +296,7 @@ def is_valid_value(self, name, value): if scalar not in compare_list: invalid.append(scalar) if len(invalid) > 0: - logger.warn("Invalid values {}".format(invalid)) + logger.warning("Invalid values {}".format(invalid)) return False # Check size of input array. diff --git a/scripts/lib/CIME/XML/pes.py b/scripts/lib/CIME/XML/pes.py index 05e077a3e95..583d308250f 100644 --- a/scripts/lib/CIME/XML/pes.py +++ b/scripts/lib/CIME/XML/pes.py @@ -112,11 +112,11 @@ def _find_matches(self, grid_nodes, grid, compset, machine, pesize_opts, overrid compset_choice = compset_match pesize_choice = pesize_match elif points == max_points: - logger.warn("mach_choice {} mach_match {}".format(mach_choice, mach_match)) - logger.warn("grid_choice {} grid_match {}".format(grid_choice, grid_match)) - logger.warn("compset_choice {} compset_match {}".format(compset_choice, compset_match)) - logger.warn("pesize_choice {} pesize_match {}".format(pesize_choice, pesize_match)) - logger.warn("points = {:d}".format(points)) + logger.warning("mach_choice {} mach_match {}".format(mach_choice, mach_match)) + logger.warning("grid_choice {} grid_match {}".format(grid_choice, grid_match)) + logger.warning("compset_choice {} compset_match {}".format(compset_choice, compset_match)) + logger.warning("pesize_choice {} pesize_match {}".format(pesize_choice, pesize_match)) + logger.warning("points = {:d}".format(points)) expect(False, "More than one PE layout matches given PE specs") if not override: for node in pe_select: diff --git a/scripts/lib/CIME/build.py b/scripts/lib/CIME/build.py index 52c1e5bcc05..16eba36a28a 100644 --- a/scripts/lib/CIME/build.py +++ b/scripts/lib/CIME/build.py @@ -324,7 +324,7 @@ def _clean_impl(case, cleanlist, clean_all): sharedlibroot = os.path.abspath(case.get_value("SHAREDLIBROOT")) expect(sharedlibroot is not None,"No SHAREDLIBROOT defined in case") if sharedlibroot != exeroot and os.path.isdir(sharedlibroot): - logging.warn("cleaning directory {}".format(sharedlibroot)) + logging.warning("cleaning directory {}".format(sharedlibroot)) shutil.rmtree(sharedlibroot) else: expect(cleanlist is not None and len(cleanlist) > 0,"Empty cleanlist not expected") diff --git a/scripts/lib/CIME/case_clone.py b/scripts/lib/CIME/case_clone.py index 9fb0d63f5d9..00980feb854 100644 --- a/scripts/lib/CIME/case_clone.py +++ b/scripts/lib/CIME/case_clone.py @@ -1,4 +1,4 @@ -import os, glob, shutil, string +import os, glob, shutil from CIME.XML.standard_module_setup import * from CIME.utils import expect from CIME.user_mod_support import apply_user_mods @@ -44,7 +44,7 @@ def create_case_clone(case, newcase, keepexe=False, mach_dir=None, project=None, olduser = case.get_value("USER") newuser = os.environ.get("USER") if olduser != newuser: - cime_output_root = string.replace(cime_output_root, olduser, newuser) + cime_output_root = cime_output_root.replace(olduser, newuser) newcase.set_value("USER", newuser) newcase.set_value("CIME_OUTPUT_ROOT", cime_output_root) diff --git a/scripts/lib/CIME/case_test.py b/scripts/lib/CIME/case_test.py index 1136a133186..0f752b5558f 100644 --- a/scripts/lib/CIME/case_test.py +++ b/scripts/lib/CIME/case_test.py @@ -47,7 +47,7 @@ def case_test(case, testname=None, reset=False): testname = case.get_value('TESTCASE') expect(testname is not None, "testname argument not resolved") - logging.warn("Running test for {}".format(testname)) + logging.warning("Running test for {}".format(testname)) _set_up_signal_handlers() diff --git a/scripts/lib/CIME/check_lockedfiles.py b/scripts/lib/CIME/check_lockedfiles.py index 1e9d87a7aae..12959afb6e8 100644 --- a/scripts/lib/CIME/check_lockedfiles.py +++ b/scripts/lib/CIME/check_lockedfiles.py @@ -71,7 +71,7 @@ def check_pelayouts_require_rebuild(case, models): new_inst = case.get_value("NINST_{}".format(comp)) if old_tasks != new_tasks or old_threads != new_threads or old_inst != new_inst: - logging.warn("{} pe change requires clean build {} {}".format(comp, old_tasks, new_tasks)) + logging.warning("{} pe change requires clean build {} {}".format(comp, old_tasks, new_tasks)) cleanflag = comp.lower() run_cmd_no_fail("./case.build --clean {}".format(cleanflag)) @@ -107,11 +107,11 @@ def check_lockedfiles(case): f1obj = case.get_env('batch') f2obj = EnvBatch(caseroot, lfile) else: - logging.warn("Locked XML file '{}' is not current being handled".format(fpart)) + logging.warning("Locked XML file '{}' is not current being handled".format(fpart)) continue diffs = f1obj.compare_xml(f2obj) if diffs: - logging.warn("File {} has been modified".format(lfile)) + logging.warning("File {} has been modified".format(lfile)) for key in diffs.keys(): print(" found difference in {} : case {} locked {}" .format(key, repr(diffs[key][0]), repr(diffs[key][1]))) @@ -122,7 +122,7 @@ def check_lockedfiles(case): expect(False, "Cannot change file env_case.xml, please" " recover the original copy from LockedFiles") elif objname == "env_build": - logging.warn("Setting build complete to False") + logging.warning("Setting build complete to False") case.set_value("BUILD_COMPLETE", False) if "PIO_VERSION" in diffs: case.set_value("BUILD_STATUS", 2) diff --git a/scripts/lib/CIME/code_checker.py b/scripts/lib/CIME/code_checker.py index 9ce8102d45f..03eac9f1347 100644 --- a/scripts/lib/CIME/code_checker.py +++ b/scripts/lib/CIME/code_checker.py @@ -16,7 +16,9 @@ def _run_pylint(on_file, interactive): ############################################################################### pylint = find_executable("pylint") - cmd_options = " --disable=I,C,R,logging-not-lazy,wildcard-import,unused-wildcard-import,fixme,broad-except,bare-except,eval-used,exec-used,global-statement,logging-format-interpolation" + cmd_options = " --disable=I,C,R,logging-not-lazy,wildcard-import,unused-wildcard-import" + cmd_options += ",fixme,broad-except,bare-except,eval-used,exec-used,global-statement" + cmd_options += ",logging-format-interpolation,no-name-in-module" cimeroot = get_cime_root() if "scripts/Tools" in on_file: @@ -69,8 +71,8 @@ def get_all_checkable_files(): cimeroot = get_cime_root() all_git_files = run_cmd_no_fail("git ls-files", from_dir=cimeroot, verbose=False).splitlines() - files_to_test = [str(item) for item in all_git_files - if ((str(item).endswith(".py") or is_python_executable(os.path.join(cimeroot, str(item)))) and not _should_pylint_skip(str(item)))] + files_to_test = [item for item in all_git_files + if ((item.endswith(".py") or is_python_executable(os.path.join(cimeroot, item))) and not _should_pylint_skip(item))] return files_to_test ############################################################################### diff --git a/scripts/lib/CIME/tests/SystemTests/test_utils/test_user_nl_utils.py b/scripts/lib/CIME/tests/SystemTests/test_utils/test_user_nl_utils.py index 67646109673..c1c6e746682 100644 --- a/scripts/lib/CIME/tests/SystemTests/test_utils/test_user_nl_utils.py +++ b/scripts/lib/CIME/tests/SystemTests/test_utils/test_user_nl_utils.py @@ -108,7 +108,7 @@ def test_append_without_files_raises_exception(self): self.write_user_nl_file(component_exists, 'irrelevant contents') # Exercise & verify - self.assertRaisesRegexp(RuntimeError, "No user_nl files found", + self.assertRaisesRegex(RuntimeError, "No user_nl files found", user_nl_utils.append_to_user_nl_files, caseroot = self._caseroot, component = component_for_append, diff --git a/scripts/lib/CIME/tests/test_user_mod_support.py b/scripts/lib/CIME/tests/test_user_mod_support.py index a1bd9b3048a..d2660eb9d46 100644 --- a/scripts/lib/CIME/tests/test_user_mod_support.py +++ b/scripts/lib/CIME/tests/test_user_mod_support.py @@ -114,7 +114,7 @@ def test_basic(self): def test_keepexe(self): self.createUserMod("foo") - with self.assertRaisesRegexp(SystemExit, "cannot have any source mods"): + with self.assertRaisesRegex(SystemExit, "cannot have any source mods"): apply_user_mods(self._caseroot, os.path.join(self._user_mods_parent_dir, "foo"), keepexe=True) diff --git a/scripts/lib/CIME/utils.py b/scripts/lib/CIME/utils.py index 99d7eb8cc3d..19fa56d8abb 100644 --- a/scripts/lib/CIME/utils.py +++ b/scripts/lib/CIME/utils.py @@ -274,13 +274,18 @@ def run_cmd(cmd, input_str=None, from_dir=None, verbose=None, env=env) output, errput = proc.communicate(input_str) - output = output.strip() if output is not None else output - errput = errput.strip() if errput is not None else errput + if output is not None: + output = output.strip() + output = output.decode("utf-8") + if errput is not None: + errput = errput.strip() + errput = errput.decode("utf-8") + stat = proc.wait() if six.PY2: - if isinstance(arg_stdout, file): + if isinstance(arg_stdout, file): # pylint: disable=undefined-variable arg_stdout.close() # pylint: disable=no-member - if isinstance(arg_stderr, file) and arg_stderr is not arg_stdout: + if isinstance(arg_stderr, file) and arg_stderr is not arg_stdout: # pylint: disable=undefined-variable arg_stderr.close() # pylint: disable=no-member else: if isinstance(arg_stdout, io.IOBase): @@ -335,7 +340,8 @@ def check_minimum_python_version(major, minor): >>> check_minimum_python_version(sys.version_info[0], sys.version_info[1]) >>> """ - expect(sys.version_info[0] == major and sys.version_info[1] >= minor, + expect(sys.version_info[0] > major or + (sys.version_info[0] == major and sys.version_info[1] >= minor), "Python {:d}, minor version {:d}+ is required, you have {:d}.{:d}".format(major, minor, sys.version_info[0], sys.version_info[1])) def normalize_case_id(case_id): @@ -1287,11 +1293,15 @@ def analyze_build_log(comp, log, compiler): logger.info("Component {} build complete with {} warnings".format(comp, warncnt)) def is_python_executable(filepath): + first_line = None if os.path.isfile(filepath): - with open(filepath, "r") as f: - first_line = f.readline() - - return first_line.startswith("#!") and "python" in first_line + with open(filepath, "rt") as f: + try: + first_line = f.readline() + except: + pass + + return first_line is not None and first_line.startswith("#!") and "python" in first_line return False def get_umask(): diff --git a/scripts/tests/list_tests b/scripts/tests/list_tests index 886cd3994ee..01b08116753 100755 --- a/scripts/tests/list_tests +++ b/scripts/tests/list_tests @@ -1,59 +1,29 @@ #!/usr/bin/env python # This script will print the list of test classes in -# scripts_regression_tests.py, however discover doesnt seem to work -# correctly in python3 so the list is hard coded. -# When run in python2 the hard coded list is checked for correctness. +# scripts_regression_tests.py # import unittest -import sys DEBUG = False #pylint: disable=protected-access def list_tests_from(): - test_classes_hard = hard_coded_list() - if sys.hexversion < 0x03000000: - loader = unittest.TestLoader() - suite = loader.discover(".", pattern="scripts_regression_tests.py") - test_classes = [] - for alltests in suite: - tests = alltests._tests - if len(tests): - for atest in tests: - if DEBUG: - print(atest) - for btest in atest._tests: - btestname = btest.__str__().split() - test_classes.append(btestname[1][1:-1].split('.')[1]) - # add this explicitly, not captured by the above - test_classes.append("B_CheckCode") - if set(test_classes) != set(test_classes_hard): - print ("Error: hard coded test_class list out of sync with actual, please fix") - sys.exit(1) - test_classes_hard = test_classes - - for ctest in sorted(list(set(test_classes_hard))): + loader = unittest.TestLoader() + suite = loader.discover(".", pattern="scripts_regression_tests.py") + test_classes = [] + for alltests in suite: + tests = alltests._tests + if len(tests): + for atest in tests: + if DEBUG: + print(atest) + for btest in atest._tests: + btestname = btest.__str__().split() + test_classes.append(btestname[1][1:-1].split('.')[1]) + # add this explicitly, not captured by the above + test_classes.append("B_CheckCode") + for ctest in sorted(list(set(test_classes))): print(ctest) -def hard_coded_list(): - return ('A_RunUnitTests', - 'B_CheckCode', - 'G_TestMacrosBasic', - 'H_TestMakeMacros', - 'I_TestCMakeMacros', - 'J_TestCreateNewcase', - 'K_TestCimeCase', - 'L_TestSaveTimings', - 'M_TestWaitForTests', - 'N_TestUnitTest', - 'O_TestTestScheduler', - 'P_TestJenkinsGenericJob', - 'Q_TestBlessTestResults', - 'S_TestManageAndQuery', - 'T_TestRunRestart', - 'X_TestSingleSubmit', - 'Z_FullSystemTest') - - if __name__ == "__main__": # Include the directories diff --git a/scripts/tests/scripts_regression_tests.py b/scripts/tests/scripts_regression_tests.py index 0e8fa3a5eb9..f48850145e6 100755 --- a/scripts/tests/scripts_regression_tests.py +++ b/scripts/tests/scripts_regression_tests.py @@ -1925,19 +1925,19 @@ class G_TestMacrosBasic(unittest.TestCase): def test_script_is_callable(self): """The test script can be called on valid output without dying.""" # This is really more a smoke test of this script than anything else. - maker = Compilers(MockMachines("mymachine", "SomeOS"), version="2.0") + maker = Compilers(MockMachines("mymachine", "SomeOS"), version=2.0) test_xml = _wrap_config_build_xml("FALSE") get_macros(maker, test_xml, "Makefile") def test_script_rejects_bad_xml(self): """The macro writer rejects input that's not valid XML.""" - maker = Compilers(MockMachines("mymachine", "SomeOS"), version="2.0") + maker = Compilers(MockMachines("mymachine", "SomeOS"), version=2.0) with self.assertRaises(ParseError): get_macros(maker, "This is not valid XML.", "Makefile") def test_script_rejects_bad_build_system(self): """The macro writer rejects a bad build system string.""" - maker = Compilers(MockMachines("mymachine", "SomeOS"), version="2.0") + maker = Compilers(MockMachines("mymachine", "SomeOS"), version=2.0) bad_string = "argle-bargle." with six.assertRaisesRegex(self, SystemExit, @@ -2334,7 +2334,7 @@ def check_for_pylint(): pylint = find_executable("pylint") if pylint is not None: output = run_cmd_no_fail("pylint --version") - pylintver = re.search(r"pylint\s+(\d+)[.](\d+)[.](\d+)", str(output)) + pylintver = re.search(r"pylint\s+(\d+)[.](\d+)[.](\d+)", output) major = int(pylintver.group(1)) minor = int(pylintver.group(2)) if pylint is None or major < 1 or (major == 1 and minor < 5): diff --git a/src/build_scripts/buildlib.internal_components b/src/build_scripts/buildlib.internal_components index 410f7e4b049..5c23073d217 100755 --- a/src/build_scripts/buildlib.internal_components +++ b/src/build_scripts/buildlib.internal_components @@ -23,7 +23,6 @@ def buildlib(caseroot, libroot, bldroot, compname=None): if dir1 == "cime_config": compname = dir2 else: - print "HERE %s"%dir1 compname = dir1.split('.')[1] build_cime_component_lib(case, compname, libroot, bldroot) diff --git a/src/build_scripts/buildlib.pio b/src/build_scripts/buildlib.pio index 5b6909b0ddf..d8bf64203a8 100755 --- a/src/build_scripts/buildlib.pio +++ b/src/build_scripts/buildlib.pio @@ -137,7 +137,7 @@ def _main_func(description): valid_values += ",pnetcdf" if netcdf4_parallel_found: valid_values += ",netcdf4p,netcdf4c" - logger.warn("Updating valid_values for PIO_TYPENAME: %s", valid_values) + logger.warning("Updating valid_values for PIO_TYPENAME: %s", valid_values) case.set_valid_values("PIO_TYPENAME",valid_values) # nothing means use the general default @@ -148,7 +148,7 @@ def _main_func(description): case.set_valid_values(comp_pio_typename,valid_values) current_value = case.get_value(comp_pio_typename) if current_value not in valid_values: - logger.warn("Resetting PIO_TYPENAME to netcdf for component %s"%comp) + logger.warning("Resetting PIO_TYPENAME to netcdf for component %s"%comp) case.set_value(comp_pio_typename,"netcdf") From 2c2ebaddf25843238dcfd4df9ecdf1c088f256fd Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 5 Oct 2017 17:36:17 -0600 Subject: [PATCH 28/41] fix doctests --- scripts/lib/CIME/aprun.py | 4 +- scripts/lib/CIME/compare_namelists.py | 2 +- scripts/lib/CIME/namelist.py | 224 ++++++++++---------------- scripts/lib/CIME/simple_compare.py | 2 +- scripts/lib/CIME/utils.py | 24 ++- scripts/lib/six.py | 15 ++ 6 files changed, 111 insertions(+), 160 deletions(-) diff --git a/scripts/lib/CIME/aprun.py b/scripts/lib/CIME/aprun.py index 8ffdd916201..97f572cd205 100755 --- a/scripts/lib/CIME/aprun.py +++ b/scripts/lib/CIME/aprun.py @@ -74,7 +74,7 @@ def _get_aprun_cmd_for_case_impl(ntasks, nthreads, rootpes, pstrids, 0, 1, maxt[0], maxt[0], 0, "" for c1 in range(1, total_tasks): if maxt[c1] != thread_count: - tasks_per_node = min(MAX_MPITASKS_PER_NODE, max_tasks_per_node / thread_count) + tasks_per_node = min(MAX_MPITASKS_PER_NODE, int(max_tasks_per_node / thread_count)) tasks_per_node = min(task_count, tasks_per_node) @@ -99,7 +99,7 @@ def _get_aprun_cmd_for_case_impl(ntasks, nthreads, rootpes, pstrids, task_count += 1 if MAX_MPITASKS_PER_NODE > 0: - tasks_per_node = min(MAX_MPITASKS_PER_NODE, max_tasks_per_node / thread_count) + tasks_per_node = min(MAX_MPITASKS_PER_NODE, int(max_tasks_per_node / thread_count)) else: tasks_per_node = max_tasks_per_node / thread_count diff --git a/scripts/lib/CIME/compare_namelists.py b/scripts/lib/CIME/compare_namelists.py index bf8f2f3af3c..d4b0fca41b6 100644 --- a/scripts/lib/CIME/compare_namelists.py +++ b/scripts/lib/CIME/compare_namelists.py @@ -425,7 +425,7 @@ def _compare_namelists(gold_namelists, comp_namelists, case): ... val3 = .false. ... /''' >>> comments = _compare_namelists(_parse_namelists(teststr1.splitlines(), 'foo'), _parse_namelists(teststr2.splitlines(), 'bar'), None) - >>> print comments + >>> print(comments) Missing namelist: nml1 Differences in namelist 'nml2': BASE: val21 = 'foo' diff --git a/scripts/lib/CIME/namelist.py b/scripts/lib/CIME/namelist.py index 5aa49166a68..acc608a262c 100644 --- a/scripts/lib/CIME/namelist.py +++ b/scripts/lib/CIME/namelist.py @@ -641,10 +641,7 @@ def literal_to_python_value(literal, type_=None): >>> literal_to_python_value("") >>> literal_to_python_value("-1.D+10") -10000000000.0 - >>> literal_to_python_value("nan(1234)") - Traceback (most recent call last): - ... - ValueError: invalid literal for float(): nan(1234) + >>> six.shouldRaise(ValueError, literal_to_python_value, "nan(1234)") """ expect(FORTRAN_REPEAT_PREFIX_REGEX.search(literal) is None, "Cannot use repetition syntax in literal_to_python_value") @@ -954,11 +951,11 @@ def get_value(self, variable_name): This function is similar to `get_variable_value`, except that it does not require a `group_name`, and it requires that the `variable_name` be unique across all groups. - - >>> parse(text='&foo bar=1 / &bazz bar=1 /').get_value('bar') + + >>> parse(text='&foo bar=1 / &bazz bar=1 /').get_value('bar') # doctest: +ELLIPSIS Traceback (most recent call last): ... - SystemExit: ERROR: Namelist.get_value: Variable {} is present in multiple groups: ['bazz', 'foo'] + SystemExit: ERROR: Namelist.get_value: Variable {} is present in multiple groups: ... >>> parse(text='&foo bar=1 / &bazz /').get_value('Bar') ['1'] >>> parse(text='&foo bar(2)=1 / &bazz /').get_value('Bar(2)') @@ -1278,10 +1275,8 @@ def _curr(self): def _next(self): """Return the character at the next position. - >>> _NamelistParser(' ')._next() - Traceback (most recent call last): - ... - _NamelistEOF: Unexpected end of file encountered in namelist. + >>> six.shouldRaise(_NamelistEOF, _NamelistParser(' ')._next) + """ # If at the end of the file, we should raise _NamelistEOF. The easiest # way to do this is to just advance. @@ -1320,14 +1315,10 @@ def _advance(self, nchars=1, check_eof=False): >>> x._advance(3) >>> (x._pos, x._line, x._col) (7, 3, 1) - >>> x._advance(1) - Traceback (most recent call last): - ... - _NamelistEOF: Unexpected end of file encountered in namelist. - >>> _NamelistParser('abc\n')._advance(4) - Traceback (most recent call last): - ... - _NamelistEOF: Unexpected end of file encountered in namelist. + >>> six.shouldRaise(_NamelistEOF, x._advance, 1) + + >>> six.shouldRaise(_NamelistEOF, _NamelistParser('abc\n')._advance, 4) + >>> x = _NamelistParser('ab') >>> x._advance(check_eof=True) False @@ -1370,10 +1361,8 @@ def _eat_whitespace(self, allow_initial_comment=False): >>> x._eat_whitespace() False >>> x._advance() - >>> x._eat_whitespace() - Traceback (most recent call last): - ... - _NamelistEOF: Unexpected end of file encountered in namelist. + >>> six.shouldRaise(_NamelistEOF, x._eat_whitespace) + >>> x = _NamelistParser(' \n! blah\n ! blah\n a') >>> x._eat_whitespace() True @@ -1427,15 +1416,11 @@ def _eat_comment(self): >>> x._curr() 'a' >>> x._advance(2) - >>> x._eat_comment() - Traceback (most recent call last): - ... - _NamelistEOF: Unexpected end of file encountered in namelist. + >>> six.shouldRaise(_NamelistEOF, x._eat_comment) + >>> x = _NamelistParser('! foo\n') - >>> x._eat_comment() - Traceback (most recent call last): - ... - _NamelistEOF: Unexpected end of file encountered in namelist. + >>> six.shouldRaise(_NamelistEOF, x._eat_comment) + """ if self._curr() != '!': return False @@ -1459,10 +1444,8 @@ def _expect_char(self, chars): >>> x = _NamelistParser('ab') >>> x._expect_char('a') >>> x._advance() - >>> x._expect_char('a') - Traceback (most recent call last): - ... - _NamelistParseError: Error in parsing namelist: expected 'a' but found 'b' + >>> six.shouldRaise(_NamelistParseError, x._expect_char, 'a') + >>> x._expect_char('ab') """ if self._curr() not in chars: @@ -1475,30 +1458,20 @@ def _expect_char(self, chars): def _parse_namelist_group_name(self): r"""Parses and returns a namelist group name at the current position. - >>> _NamelistParser('abc')._parse_namelist_group_name() - Traceback (most recent call last): - ... - _NamelistParseError: Error in parsing namelist: expected '&' but found 'a' - >>> _NamelistParser('&abc')._parse_namelist_group_name() - Traceback (most recent call last): - ... - _NamelistEOF: Unexpected end of file encountered in namelist. + >>> six.shouldRaise(_NamelistParseError, _NamelistParser('abc')._parse_namelist_group_name) + + >>> six.shouldRaise(_NamelistEOF, _NamelistParser('&abc')._parse_namelist_group_name) + >>> _NamelistParser('&abc ')._parse_namelist_group_name() 'abc' >>> _NamelistParser('&abc\n')._parse_namelist_group_name() 'abc' - >>> _NamelistParser('&abc/ ')._parse_namelist_group_name() - Traceback (most recent call last): - ... - _NamelistParseError: Error in parsing namelist: 'abc/' is not a valid variable name - >>> _NamelistParser('&abc= ')._parse_namelist_group_name() - Traceback (most recent call last): - ... - _NamelistParseError: Error in parsing namelist: 'abc=' is not a valid variable name - >>> _NamelistParser('& ')._parse_namelist_group_name() - Traceback (most recent call last): - ... - _NamelistParseError: Error in parsing namelist: '' is not a valid variable name + >>> six.shouldRaise(_NamelistParseError, _NamelistParser('&abc/ ')._parse_namelist_group_name) + + >>> six.shouldRaise(_NamelistParseError, _NamelistParser('&abc= ')._parse_namelist_group_name) + + >>> six.shouldRaise(_NamelistParseError, _NamelistParser('& ')._parse_namelist_group_name) + """ self._expect_char("&") self._advance() @@ -1511,10 +1484,8 @@ def _parse_variable_name(self, allow_equals=True): variable name; if it is `False`, only white space can be used for this purpose. - >>> _NamelistParser('abc')._parse_variable_name() - Traceback (most recent call last): - ... - _NamelistEOF: Unexpected end of file encountered in namelist. + >>> six.shouldRaise(_NamelistEOF, _NamelistParser('abc')._parse_variable_name) + >>> _NamelistParser('foo(2)= ')._parse_variable_name() 'foo(2)' >>> _NamelistParser('abc ')._parse_variable_name() @@ -1531,18 +1502,21 @@ def _parse_variable_name(self, allow_equals=True): 'abc(1:2:3)' >>> _NamelistParser('abc=')._parse_variable_name() 'abc' - >>> _NamelistParser('abc(1,2) ')._parse_variable_name() - Traceback (most recent call last): - ... - _NamelistParseError: Error in parsing namelist: Multiple dimensions not supported in CIME namelist variables 'abc(1,2)' - >>> _NamelistParser('abc, ')._parse_variable_name() - Traceback (most recent call last): - ... - _NamelistParseError: Error in parsing namelist: 'abc,' is not a valid variable name - >>> _NamelistParser(' ')._parse_variable_name() - Traceback (most recent call last): - ... - _NamelistParseError: Error in parsing namelist: '' is not a valid variable name + >>> try: + ... _NamelistParser('abc(1,2) ')._parse_variable_name() + ... raise AssertionError("_NamelistParseError not raised") + ... except _NamelistParseError: + ... pass + >>> try: + ... _NamelistParser('abc, ')._parse_variable_name() + ... raise AssertionError("_NamelistParseError not raised") + ... except _NamelistParseError: + ... pass + >>> try: + ... _NamelistParser(' ')._parse_variable_name() + ... raise AssertionError("_NamelistParseError not raised") + ... except _NamelistParseError: + ... pass >>> _NamelistParser('foo+= ')._parse_variable_name() 'foo' """ @@ -1582,18 +1556,14 @@ def _parse_character_literal(self): Position on return is the last character of the string; we avoid advancing past that in order to avoid potential EOF errors. - >>> _NamelistParser('"abc')._parse_character_literal() - Traceback (most recent call last): - ... - _NamelistEOF: Unexpected end of file encountered in namelist. + >>> six.shouldRaise(_NamelistEOF, _NamelistParser('"abc')._parse_character_literal) + >>> _NamelistParser('"abc" ')._parse_character_literal() '"abc"' >>> _NamelistParser("'abc' ")._parse_character_literal() "'abc'" - >>> _NamelistParser("*abc* ")._parse_character_literal() - Traceback (most recent call last): - ... - _NamelistParseError: Error in parsing namelist: *abc* is not a valid character literal + >>> six.shouldRaise(_NamelistParseError, _NamelistParser("*abc* ")._parse_character_literal) + >>> _NamelistParser("'abc''def' ")._parse_character_literal() "'abc''def'" >>> _NamelistParser("'abc''' ")._parse_character_literal() @@ -1626,16 +1596,12 @@ def _parse_complex_literal(self): Position on return is the last character of the string; we avoid advancing past that in order to avoid potential EOF errors. - >>> _NamelistParser('(1.,2.')._parse_complex_literal() - Traceback (most recent call last): - ... - _NamelistEOF: Unexpected end of file encountered in namelist. + >>> six.shouldRaise(_NamelistEOF, _NamelistParser('(1.,2.')._parse_complex_literal) + >>> _NamelistParser('(1.,2.) ')._parse_complex_literal() '(1.,2.)' - >>> _NamelistParser("(A,B) ")._parse_complex_literal() - Traceback (most recent call last): - ... - _NamelistParseError: Error in parsing namelist: '(A,B)' is not a valid complex literal + >>> six.shouldRaise(_NamelistParseError, _NamelistParser("(A,B) ")._parse_complex_literal) + """ old_pos = self._pos while self._curr() != ')': @@ -1714,18 +1680,14 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): '"abc"' >>> _NamelistParser("'abc' ")._parse_literal() "'abc'" - >>> _NamelistParser('"abc"')._parse_literal() - Traceback (most recent call last): - ... - _NamelistEOF: Unexpected end of file encountered in namelist. + >>> six.shouldRaise(_NamelistEOF, _NamelistParser('"abc"')._parse_literal) + >>> _NamelistParser('"abc"')._parse_literal(allow_eof_end=True) '"abc"' >>> _NamelistParser('(1.,2.) ')._parse_literal() '(1.,2.)' - >>> _NamelistParser('(1.,2.)')._parse_literal() - Traceback (most recent call last): - ... - _NamelistEOF: Unexpected end of file encountered in namelist. + >>> six.shouldRaise(_NamelistEOF, _NamelistParser('(1.,2.)')._parse_literal) + >>> _NamelistParser('(1.,2.)')._parse_literal(allow_eof_end=True) '(1.,2.)' >>> _NamelistParser('5 ')._parse_literal() @@ -1738,10 +1700,8 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): 'nan(booga)' >>> _NamelistParser('.FLORIDA$ ')._parse_literal() '.FLORIDA$' - >>> _NamelistParser('hamburger ')._parse_literal() - Traceback (most recent call last): - ... - _NamelistParseError: Error in parsing namelist: expected literal value, but got 'hamburger' + >>> six.shouldRaise(_NamelistParseError, _NamelistParser('hamburger ')._parse_literal) + >>> _NamelistParser('5,')._parse_literal() '5' >>> _NamelistParser('5\n')._parse_literal() @@ -1756,20 +1716,14 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): '6*(1., 2.)' >>> _NamelistParser('6*"a" ')._parse_literal() '6*"a"' - >>> _NamelistParser('6*')._parse_literal() - Traceback (most recent call last): - ... - _NamelistEOF: Unexpected end of file encountered in namelist. + >>> six.shouldRaise(_NamelistEOF, _NamelistParser('6*')._parse_literal) + >>> _NamelistParser('6*')._parse_literal(allow_eof_end=True) '6*' - >>> _NamelistParser('foo= ')._parse_literal() - Traceback (most recent call last): - ... - _NamelistParseError: Error in parsing namelist: expected literal value, but got 'foo=' - >>> _NamelistParser('foo+= ')._parse_literal() - Traceback (most recent call last): - ... - _NamelistParseError: Error in parsing namelist: expected literal value, but got 'foo+=' + >>> six.shouldRaise(_NamelistParseError, _NamelistParser('foo= ')._parse_literal) + + >>> six.shouldRaise(_NamelistParseError, _NamelistParser('foo+= ')._parse_literal) + >>> _NamelistParser('5,')._parse_literal(allow_name=True) '5' >>> x = _NamelistParser('foo= ') @@ -1780,14 +1734,10 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): >>> x._parse_literal(allow_name=True) >>> x._curr() 'f' - >>> _NamelistParser('6*foo= ')._parse_literal(allow_name=True) - Traceback (most recent call last): - ... - _NamelistParseError: Error in parsing namelist: expected literal value, but got '6*foo=' - >>> _NamelistParser('6*foo+= ')._parse_literal(allow_name=True) - Traceback (most recent call last): - ... - _NamelistParseError: Error in parsing namelist: expected literal value, but got '6*foo+=' + >>> six.shouldRaise(_NamelistParseError, _NamelistParser('6*foo= ')._parse_literal, allow_name=True) + + >>> six.shouldRaise(_NamelistParseError, _NamelistParser('6*foo+= ')._parse_literal, allow_name=True) + >>> x = _NamelistParser('foo = ') >>> x._parse_literal(allow_name=True) >>> x._curr() @@ -1889,10 +1839,8 @@ def _expect_separator(self, allow_eof=False): >>> x._curr() '/' >>> x = _NamelistParser("a") - >>> x._expect_separator() - Traceback (most recent call last): - ... - _NamelistParseError: Error in parsing namelist: expected one of the characters in ' \n,/' but found 'a' + >>> six.shouldRaise(_NamelistParseError, x._expect_separator) + >>> x = _NamelistParser(" , a") >>> x._expect_separator() True @@ -1922,10 +1870,8 @@ def _expect_separator(self, allow_eof=False): >>> x._expect_separator(allow_eof=True) True >>> x = _NamelistParser(" / ") - >>> x._expect_separator(allow_eof=True) - Traceback (most recent call last): - ... - _NamelistParseError: Error in parsing namelist: found group-terminating '/' in file without group names + >>> six.shouldRaise(_NamelistParseError, x._expect_separator, allow_eof=True) + """ errstring = "found group-terminating '/' in file without group names" # Deal with the possibility that we are already at EOF. @@ -1974,10 +1920,8 @@ def _parse_name_and_values(self, allow_eof_end=False): ('foo', ["'bar'"], False) >>> _NamelistParser("foo=\n'bar' /")._parse_name_and_values() ('foo', ["'bar'"], False) - >>> _NamelistParser("foo 'bar' /")._parse_name_and_values() - Traceback (most recent call last): - ... - _NamelistParseError: Error in parsing namelist: expected '=' but found "'" + >>> six.shouldRaise(_NamelistParseError, _NamelistParser("foo 'bar' /")._parse_name_and_values) + >>> _NamelistParser("foo='bar','bazz' /")._parse_name_and_values() ('foo', ["'bar'", "'bazz'"], False) >>> _NamelistParser("foo=,,'bazz',6*/")._parse_name_and_values() @@ -1986,18 +1930,14 @@ def _parse_name_and_values(self, allow_eof_end=False): ('foo', ["'bar'", "'bazz'"], False) >>> _NamelistParser("foo='bar' 'bazz' foo2(2)='ban'")._parse_name_and_values() ('foo', ["'bar'", "'bazz'"], False) - >>> _NamelistParser("foo= foo2='ban' ")._parse_name_and_values() - Traceback (most recent call last): - ... - _NamelistParseError: Error in parsing namelist: expected literal value, but got "foo2='ban'" + >>> six.shouldRaise(_NamelistParseError, _NamelistParser("foo= foo2='ban' ")._parse_name_and_values) + >>> _NamelistParser("foo=,,'bazz',6* ")._parse_name_and_values(allow_eof_end=True) ('foo', ['', '', "'bazz'", '6*'], False) >>> _NamelistParser("foo(3)='bazz'")._parse_name_and_values(allow_eof_end=True) ('foo(3)', ["'bazz'"], False) - >>> _NamelistParser("foo=")._parse_name_and_values() - Traceback (most recent call last): - ... - _NamelistEOF: Unexpected end of file encountered in namelist. + >>> six.shouldRaise(_NamelistEOF, _NamelistParser("foo=")._parse_name_and_values) + >>> _NamelistParser("foo=")._parse_name_and_values(allow_eof_end=True) ('foo', [''], False) >>> _NamelistParser("foo= ")._parse_name_and_values(allow_eof_end=True) @@ -2076,10 +2016,8 @@ def _parse_namelist_group(self): >>> x = _NamelistParser("&group /&group /") >>> x._parse_namelist_group() >>> x._advance() - >>> x._parse_namelist_group() - Traceback (most recent call last): - ... - _NamelistParseError: Error in parsing namelist: Namelist group 'group' encountered twice. + >>> six.shouldRaise(_NamelistParseError, x._parse_namelist_group) + >>> x = _NamelistParser("&group foo='bar', foo='bazz' /") >>> x._parse_namelist_group() >>> x._settings diff --git a/scripts/lib/CIME/simple_compare.py b/scripts/lib/CIME/simple_compare.py index 086297b4d1b..5d86b175764 100644 --- a/scripts/lib/CIME/simple_compare.py +++ b/scripts/lib/CIME/simple_compare.py @@ -65,7 +65,7 @@ def _compare_data(gold_lines, comp_lines, case): ... data00 ... ''' >>> results = _compare_data(teststr.splitlines(), teststr2.splitlines(), None) - >>> print results + >>> print(results) Inequivalent lines data2 data3 != data2 data30 NORMALIZED: data2 data3 != data2 data30 Found extra lines diff --git a/scripts/lib/CIME/utils.py b/scripts/lib/CIME/utils.py index 19fa56d8abb..bd88dd598fe 100644 --- a/scripts/lib/CIME/utils.py +++ b/scripts/lib/CIME/utils.py @@ -310,19 +310,17 @@ def run_cmd_no_fail(cmd, input_str=None, from_dir=None, verbose=None, Wrapper around subprocess to make it much more convenient to run shell commands. Expects command to work. Just returns output string. - >>> run_cmd_no_fail('echo foo') - 'foo' - + >>> run_cmd_no_fail('echo foo') == 'foo' + True >>> run_cmd_no_fail('echo THE ERROR >&2; false') # doctest:+ELLIPSIS Traceback (most recent call last): ... SystemExit: ERROR: Command: 'echo THE ERROR >&2; false' failed with error 'THE ERROR' from dir ... - >>> run_cmd_no_fail('grep foo', input_str='foo') - 'foo' - - >>> run_cmd_no_fail('echo THE ERROR >&2', combine_output=True) - 'THE ERROR' + >>> run_cmd_no_fail('grep foo', input_str=b'foo') == 'foo' + True + >>> run_cmd_no_fail('echo THE ERROR >&2', combine_output=True) == 'THE ERROR' + True """ stat, output, errput = run_cmd(cmd, input_str, from_dir, verbose, arg_stdout, arg_stderr, env, combine_output) if stat != 0: @@ -913,9 +911,9 @@ def convert_to_babylonian_time(seconds): >>> convert_to_babylonian_time(3661) '01:01:01' """ - hours = seconds / 3600 + hours = int(seconds / 3600) seconds %= 3600 - minutes = seconds / 60 + minutes = int(seconds / 60) seconds %= 60 return "{:02d}:{:02d}:{:02d}".format(hours, minutes, seconds) @@ -1160,11 +1158,11 @@ def gzip_existing_file(filepath): >>> import tempfile >>> fd, filename = tempfile.mkstemp(text=True) - >>> _ = os.write(fd, "Hello World") + >>> _ = os.write(fd, b"Hello World") >>> os.close(fd) >>> gzfile = gzip_existing_file(filename) - >>> gunzip_existing_file(gzfile) - 'Hello World' + >>> gunzip_existing_file(gzfile) == b'Hello World' + True >>> os.remove(gzfile) """ expect(os.path.exists(filepath), "{} does not exists".format(filepath)) diff --git a/scripts/lib/six.py b/scripts/lib/six.py index 6bf4fd38104..5e5ae0eef87 100644 --- a/scripts/lib/six.py +++ b/scripts/lib/six.py @@ -866,6 +866,21 @@ def python_2_unicode_compatible(klass): return klass +def shouldRaise(eclass, method, *args, **kw): + """ + http://python3porting.com/problems.html#running-doctests + """ + try: + method(*args, **kw) + except: + e = sys.exc_info()[1] + if not isinstance(e, eclass): + raise + return + raise Exception("Expected exception %s not raised" % + str(eclass)) + + # Complete the moves implementation. # This code is at the end of this module to speed up module loading. # Turn this module into a package. From 15f110f798625899a2e84eefb2720fe870494ae7 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 5 Oct 2017 17:41:55 -0600 Subject: [PATCH 29/41] version should be int --- scripts/tests/scripts_regression_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/scripts_regression_tests.py b/scripts/tests/scripts_regression_tests.py index f48850145e6..de746573829 100755 --- a/scripts/tests/scripts_regression_tests.py +++ b/scripts/tests/scripts_regression_tests.py @@ -1961,7 +1961,7 @@ class H_TestMakeMacros(unittest.TestCase): test_machine = "mymachine" def setUp(self): - self._maker = Compilers(MockMachines(self.test_machine, self.test_os), version="2.0") + self._maker = Compilers(MockMachines(self.test_machine, self.test_os), version=2.0) def xml_to_tester(self, xml_string): """Helper that directly converts an XML string to a MakefileTester.""" From 7ab3a399de77ba4d9395f44fcaf82fa660999116 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 5 Oct 2017 18:07:10 -0600 Subject: [PATCH 30/41] more py3 fixes --- scripts/lib/CIME/XML/env_base.py | 4 ++-- scripts/lib/CIME/XML/env_mach_pes.py | 4 +++- scripts/lib/CIME/case.py | 2 +- scripts/lib/CIME/case_setup.py | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/lib/CIME/XML/env_base.py b/scripts/lib/CIME/XML/env_base.py index 21bc79fbf6d..e5fa2e33830 100644 --- a/scripts/lib/CIME/XML/env_base.py +++ b/scripts/lib/CIME/XML/env_base.py @@ -48,9 +48,9 @@ def check_if_comp_var(self, vid, attribute=None): new_vid = None for comp in self._components: if "_"+comp in vid: - new_vid = string.replace(vid, '_'+comp, '', 1) + new_vid = vid.replace('_'+comp, '', 1) elif comp+"_" in vid: - new_vid = string.replace(vid, comp+'_', '', 1) + new_vid = vid.replace(comp+'_', '', 1) if new_vid is not None: break diff --git a/scripts/lib/CIME/XML/env_mach_pes.py b/scripts/lib/CIME/XML/env_mach_pes.py index 172965fce09..1a091ccbfbb 100644 --- a/scripts/lib/CIME/XML/env_mach_pes.py +++ b/scripts/lib/CIME/XML/env_mach_pes.py @@ -88,7 +88,9 @@ def get_total_tasks(self, comp_classes): ntasks = self.get_value("NTASKS", attribute={"component":comp}) rootpe = self.get_value("ROOTPE", attribute={"component":comp}) pstrid = self.get_value("PSTRID", attribute={"component":comp}) - maxinst = max(maxinst, self.get_value("NINST", attribute={"component":comp})) + if comp != "CPL": + ninst = self.get_value("NINST", attribute={"component":comp}) + maxinst = max(maxinst, ninst) tt = rootpe + (ntasks - 1) * pstrid + 1 total_tasks = max(tt, total_tasks) if self.get_value("MULTI_DRIVER"): diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index e481b8aea59..cc96df01db4 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -411,7 +411,7 @@ def set_lookup_value(self, item, value): def clean_up_lookups(self, allow_undefined=False): # put anything in the lookups table into existing env objects - for key,value in self.lookups.items(): + for key,value in list(self.lookups.items()): logger.debug("lookup key {} value {}".format(key, value)) result = self.set_value(key,value, allow_undefined=allow_undefined) if result is not None: diff --git a/scripts/lib/CIME/case_setup.py b/scripts/lib/CIME/case_setup.py index ad71e8482ca..532a250f0d3 100644 --- a/scripts/lib/CIME/case_setup.py +++ b/scripts/lib/CIME/case_setup.py @@ -144,7 +144,7 @@ def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False): ntasks = ninst else: expect(False, "NINST_{} value {:d} greater than NTASKS_{} {:d}".format(comp, ninst, comp, ntasks)) - case.set_value("NTASKS_PER_INST_{}".format(comp), ntasks / ninst) + case.set_value("NTASKS_PER_INST_{}".format(comp), int(ntasks / ninst)) if os.path.exists("case.run"): logger.info("Machine/Decomp/Pes configuration has already been done ...skipping") From fbe73f973e2a3d6da884d972e568bf63d0fc22cb Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 5 Oct 2017 19:29:29 -0600 Subject: [PATCH 31/41] fix another issue --- scripts/lib/CIME/buildlib.py | 7 ++----- scripts/lib/CIME/utils.py | 12 ++++++++---- src/drivers/mct/cime_config/buildnml | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/scripts/lib/CIME/buildlib.py b/scripts/lib/CIME/buildlib.py index 35d5baf94fc..a2f97fc6a93 100644 --- a/scripts/lib/CIME/buildlib.py +++ b/scripts/lib/CIME/buildlib.py @@ -6,7 +6,7 @@ from CIME.case import Case from CIME.utils import parse_args_and_handle_standard_logging_options, setup_standard_logging_options import sys, os, argparse, doctest - +import six logger = logging.getLogger(__name__) ############################################################################### @@ -86,7 +86,4 @@ def run_gmake(case, compclass, libroot, bldroot, libname="", user_cppdefs=""): if user_cppdefs: cmd = cmd + "USER_CPPDEFS='{}'".format(user_cppdefs ) - rc, out, err = run_cmd(cmd) - expect(rc == 0, "Command {} failed rc={:d}\nout={}\nerr={}".format(cmd, rc, out, err)) - - print("Command {} completed with output {}\nerr {}".format(cmd, out, err)) + run_cmd_no_fail(cmd, combine_output=True) diff --git a/scripts/lib/CIME/utils.py b/scripts/lib/CIME/utils.py index bd88dd598fe..f703b9740ef 100644 --- a/scripts/lib/CIME/utils.py +++ b/scripts/lib/CIME/utils.py @@ -275,11 +275,15 @@ def run_cmd(cmd, input_str=None, from_dir=None, verbose=None, output, errput = proc.communicate(input_str) if output is not None: - output = output.strip() - output = output.decode("utf-8") + try: + output = output.decode('utf-8').strip() + except AttributeError: + pass if errput is not None: - errput = errput.strip() - errput = errput.decode("utf-8") + try: + errput = errput.decode('utf-8').strip() + except AttributeError: + pass stat = proc.wait() if six.PY2: diff --git a/src/drivers/mct/cime_config/buildnml b/src/drivers/mct/cime_config/buildnml index 8ce75173335..970efa691c0 100755 --- a/src/drivers/mct/cime_config/buildnml +++ b/src/drivers/mct/cime_config/buildnml @@ -99,7 +99,7 @@ def _create_drv_namelists(case, infile, confdir, nmlgen, files): for comp in comps: ncpl = case.get_value(comp.upper() + '_NCPL') if ncpl is not None: - cpl_dt = basedt / int(ncpl) + cpl_dt = int(basedt / int(ncpl)) totaldt = cpl_dt * int(ncpl) if totaldt != basedt: expect(False, " {} ncpl doesn't divide base dt evenly".format(comp)) From 05e82c39262475f4c19841a69949b0e2149d12ca Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 5 Oct 2017 19:46:35 -0600 Subject: [PATCH 32/41] more py3 issues --- scripts/lib/CIME/XML/generic_xml.py | 4 ++-- .../CIME/tests/SystemTests/test_utils/test_user_nl_utils.py | 3 ++- scripts/lib/CIME/tests/test_user_mod_support.py | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/lib/CIME/XML/generic_xml.py b/scripts/lib/CIME/XML/generic_xml.py index 836298af0b9..0ba9f44b336 100644 --- a/scripts/lib/CIME/XML/generic_xml.py +++ b/scripts/lib/CIME/XML/generic_xml.py @@ -197,8 +197,8 @@ def get_resolved_value(self, raw_value): '4' >>> obj.get_resolved_value("0001-01-01") '0001-01-01' - >>> obj.get_resolved_value("$SHELL{echo hi}") - 'hi' + >>> obj.get_resolved_value("$SHELL{echo hi}") == 'hi' + True """ logger.debug("raw_value {}".format(raw_value)) reference_re = re.compile(r'\${?(\w+)}?') diff --git a/scripts/lib/CIME/tests/SystemTests/test_utils/test_user_nl_utils.py b/scripts/lib/CIME/tests/SystemTests/test_utils/test_user_nl_utils.py index c1c6e746682..9164530e3f0 100644 --- a/scripts/lib/CIME/tests/SystemTests/test_utils/test_user_nl_utils.py +++ b/scripts/lib/CIME/tests/SystemTests/test_utils/test_user_nl_utils.py @@ -5,6 +5,7 @@ import shutil import tempfile from CIME.SystemTests.test_utils import user_nl_utils +import six class TestUserNLCopier(unittest.TestCase): @@ -108,7 +109,7 @@ def test_append_without_files_raises_exception(self): self.write_user_nl_file(component_exists, 'irrelevant contents') # Exercise & verify - self.assertRaisesRegex(RuntimeError, "No user_nl files found", + six.assertRaisesRegex(self, RuntimeError, "No user_nl files found", user_nl_utils.append_to_user_nl_files, caseroot = self._caseroot, component = component_for_append, diff --git a/scripts/lib/CIME/tests/test_user_mod_support.py b/scripts/lib/CIME/tests/test_user_mod_support.py index d2660eb9d46..75bcf33f03f 100644 --- a/scripts/lib/CIME/tests/test_user_mod_support.py +++ b/scripts/lib/CIME/tests/test_user_mod_support.py @@ -5,7 +5,7 @@ import tempfile import os from CIME.user_mod_support import apply_user_mods - +import six # ======================================================================== # Define some parameters # ======================================================================== @@ -114,7 +114,7 @@ def test_basic(self): def test_keepexe(self): self.createUserMod("foo") - with self.assertRaisesRegex(SystemExit, "cannot have any source mods"): + with six.assertRaisesRegex(self, SystemExit, "cannot have any source mods"): apply_user_mods(self._caseroot, os.path.join(self._user_mods_parent_dir, "foo"), keepexe=True) From 6355cd8a2718f5a4d7fdc3a334229f1ac3e10629 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 5 Oct 2017 19:57:36 -0600 Subject: [PATCH 33/41] another py3 fix --- scripts/lib/CIME/XML/env_mach_pes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/lib/CIME/XML/env_mach_pes.py b/scripts/lib/CIME/XML/env_mach_pes.py index 1a091ccbfbb..c70902ba9a1 100644 --- a/scripts/lib/CIME/XML/env_mach_pes.py +++ b/scripts/lib/CIME/XML/env_mach_pes.py @@ -25,7 +25,8 @@ def get_value(self, vid, attribute=None, resolved=True, subgroup=None, MAX_MPITA if vid == "NINST_MAX": value = 1 for comp in self._components: - value = max(value, self.get_value("NINST_{}".format(comp))) + if comp != "CPL": + value = max(value, self.get_value("NINST_{}".format(comp))) return value value = EnvBase.get_value(self, vid, attribute, resolved, subgroup) From 90ddf5bf0b7bc891c88c8e018eba07270fcf2d38 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 5 Oct 2017 20:32:31 -0600 Subject: [PATCH 34/41] another type conv problem --- scripts/lib/CIME/SystemTests/ers.py | 2 +- scripts/lib/CIME/SystemTests/system_tests_common.py | 9 +++++---- scripts/tests/scripts_regression_tests.py | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/lib/CIME/SystemTests/ers.py b/scripts/lib/CIME/SystemTests/ers.py index 28698084c1d..2de668606a0 100644 --- a/scripts/lib/CIME/SystemTests/ers.py +++ b/scripts/lib/CIME/SystemTests/ers.py @@ -28,7 +28,7 @@ def _ers_second_phase(self): stop_n = self._case.get_value("STOP_N") stop_option = self._case.get_value("STOP_OPTION") - rest_n = stop_n/2 + 1 + rest_n = int(stop_n/2 + 1) stop_new = stop_n - rest_n expect(stop_new > 0, "ERROR: stop_n value {:d} too short {:d} {:d}".format(stop_new,stop_n,rest_n)) diff --git a/scripts/lib/CIME/SystemTests/system_tests_common.py b/scripts/lib/CIME/SystemTests/system_tests_common.py index eeaa9997bab..ecff0cd362a 100644 --- a/scripts/lib/CIME/SystemTests/system_tests_common.py +++ b/scripts/lib/CIME/SystemTests/system_tests_common.py @@ -13,7 +13,7 @@ import CIME.build as build -import shutil, glob, gzip, time, traceback +import shutil, glob, gzip, time, traceback, six logger = logging.getLogger(__name__) @@ -239,10 +239,11 @@ def _coupler_log_indicates_run_complete(self): allgood = len(newestcpllogfiles) for cpllog in newestcpllogfiles: try: - if "SUCCESSFUL TERMINATION" in gzip.open(cpllog, 'rb').read(): + if six.b("SUCCESSFUL TERMINATION") in gzip.open(cpllog, 'rb').read(): allgood = allgood - 1 - except: - logger.info("{} is not compressed, assuming run failed".format(cpllog)) + except BaseException as msg: + + logger.info("{} is not compressed, assuming run failed {}".format(cpllog, msg)) return allgood==0 diff --git a/scripts/tests/scripts_regression_tests.py b/scripts/tests/scripts_regression_tests.py index de746573829..8a80a55ad7d 100755 --- a/scripts/tests/scripts_regression_tests.py +++ b/scripts/tests/scripts_regression_tests.py @@ -747,7 +747,7 @@ def setUp(self): self._baseline_area = os.path.join(TEST_ROOT, "baselines") self._testroot = TEST_ROOT self._hasbatch = MACHINE.has_batch_system() and not NO_BATCH - self._do_teardown = True # Will never do teardown if test failed + self._do_teardown = False # Will never do teardown if test failed ########################################################################### def tearDown(self): From 70e039af7236b1ce4730443c9eed5af29a616a7b Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 5 Oct 2017 20:41:56 -0600 Subject: [PATCH 35/41] bin to str issues --- scripts/lib/CIME/SystemTests/system_tests_common.py | 4 ++-- scripts/tests/scripts_regression_tests.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/lib/CIME/SystemTests/system_tests_common.py b/scripts/lib/CIME/SystemTests/system_tests_common.py index ecff0cd362a..4d5e9a52b9c 100644 --- a/scripts/lib/CIME/SystemTests/system_tests_common.py +++ b/scripts/lib/CIME/SystemTests/system_tests_common.py @@ -281,7 +281,7 @@ def _get_mem_usage(self, cpllog): fopen = open with fopen(cpllog, "rb") as f: for line in f: - m = meminfo.match(line) + m = meminfo.match(line.decode('utf-8')) if m: memlist.append((float(m.group(1)), float(m.group(2)))) # Remove the last mem record, it's sometimes artificially high @@ -296,7 +296,7 @@ def _get_throughput(self, cpllog): """ if cpllog is not None and os.path.isfile(cpllog): with gzip.open(cpllog, "rb") as f: - cpltext = f.read() + cpltext = f.read().decode('utf-8') m = re.search(r"# simulated years / cmp-day =\s+(\d+\.\d+)\s",cpltext) if m: return float(m.group(1)) diff --git a/scripts/tests/scripts_regression_tests.py b/scripts/tests/scripts_regression_tests.py index 8a80a55ad7d..de746573829 100755 --- a/scripts/tests/scripts_regression_tests.py +++ b/scripts/tests/scripts_regression_tests.py @@ -747,7 +747,7 @@ def setUp(self): self._baseline_area = os.path.join(TEST_ROOT, "baselines") self._testroot = TEST_ROOT self._hasbatch = MACHINE.has_batch_system() and not NO_BATCH - self._do_teardown = False # Will never do teardown if test failed + self._do_teardown = True # Will never do teardown if test failed ########################################################################### def tearDown(self): From cfbe500095ccb2201697962bf84adc11c877f2e5 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 5 Oct 2017 21:19:29 -0600 Subject: [PATCH 36/41] make tests work on congo --- config/cesm/machines/config_machines.xml | 2 +- scripts/lib/CIME/get_timing.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/config/cesm/machines/config_machines.xml b/config/cesm/machines/config_machines.xml index 755fbbf5c46..8c55b92d2a3 100644 --- a/config/cesm/machines/config_machines.xml +++ b/config/cesm/machines/config_machines.xml @@ -214,7 +214,7 @@ none me@my.address 8 - 4 + 8 FALSE mpiexec diff --git a/scripts/lib/CIME/get_timing.py b/scripts/lib/CIME/get_timing.py index 4353f78a860..4ae32edbce4 100644 --- a/scripts/lib/CIME/get_timing.py +++ b/scripts/lib/CIME/get_timing.py @@ -125,8 +125,11 @@ def _getTiming(self, inst=0): ncpl_base_period = self.case.get_value("NCPL_BASE_PERIOD") ncpl = 0 for compclass in self.case.get_values("COMP_CLASSES"): - ncpl = max(ncpl, self.case.get_value("{}_NCPL".format(compclass))) - ocn_ncpl = self.case.get_value("OCN_NCPL") + comp_ncpl = self.case.get_value("{}_NCPL".format(compclass)) + if compclass == "OCN": + ocn_ncpl = comp_ncpl + if comp_ncpl is not None: + ncpl = max(ncpl, comp_ncpl) compset = self.case.get_value("COMPSET") if compset is None: From db0d7f961e67eb8870a6bcba1b9d1746a72dc9dc Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 5 Oct 2017 22:01:57 -0600 Subject: [PATCH 37/41] fix some division --- scripts/lib/CIME/SystemTests/dae.py | 2 +- scripts/lib/CIME/SystemTests/eri.py | 6 +++--- scripts/lib/CIME/SystemTests/pet.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/lib/CIME/SystemTests/dae.py b/scripts/lib/CIME/SystemTests/dae.py index 5e70a6b2c5b..cdd062c9c04 100644 --- a/scripts/lib/CIME/SystemTests/dae.py +++ b/scripts/lib/CIME/SystemTests/dae.py @@ -54,7 +54,7 @@ def _case_two_setup(self): self._case.set_value("DATA_ASSIMILATION_CYCLES", 2) stopn = self._case.get_value("STOP_N") expect((stopn % 2) == 0, "ERROR: DAE test requires that STOP_N be even") - stopn = stopn / 2 + stopn = int(stopn / 2) self._case.set_value("STOP_N", stopn) self._case.flush() diff --git a/scripts/lib/CIME/SystemTests/eri.py b/scripts/lib/CIME/SystemTests/eri.py index 955b1918019..55446269aa2 100644 --- a/scripts/lib/CIME/SystemTests/eri.py +++ b/scripts/lib/CIME/SystemTests/eri.py @@ -52,12 +52,12 @@ def run_phase(self): stop_option = self._case.get_value("STOP_OPTION") run_startdate = self._case.get_value("RUN_STARTDATE") - stop_n1 = stop_n / 6 + stop_n1 = int(stop_n / 6) rest_n1 = stop_n1 start_1 = run_startdate stop_n2 = stop_n - stop_n1 - rest_n2 = stop_n2 / 2 + 1 + rest_n2 = int(stop_n2 / 2 + 1) hist_n = stop_n2 start_1_year, start_1_month, start_1_day = [int(item) for item in start_1.split("-")] @@ -65,7 +65,7 @@ def run_phase(self): start_2 = "{:04d}-{:02d}-{:02d}".format(start_2_year, start_1_month, start_1_day) stop_n3 = stop_n2 - rest_n2 - rest_n3 = stop_n3 / 2 + 1 + rest_n3 = int(stop_n3 / 2 + 1) stop_n4 = stop_n3 - rest_n3 diff --git a/scripts/lib/CIME/SystemTests/pet.py b/scripts/lib/CIME/SystemTests/pet.py index fe4a526db7f..1cb15939f39 100644 --- a/scripts/lib/CIME/SystemTests/pet.py +++ b/scripts/lib/CIME/SystemTests/pet.py @@ -45,7 +45,7 @@ def _case_two_setup(self): # machines, if the mpiexec tries to exceed the procs-per-node that were given # to the batch submission, things break. Setting MAX_TASKS_PER_NODE to half of # it original value prevents this. - self._case.set_value("MAX_TASKS_PER_NODE", self._case.get_value("MAX_TASKS_PER_NODE") / 2) + self._case.set_value("MAX_TASKS_PER_NODE", int(self._case.get_value("MAX_TASKS_PER_NODE") / 2)) # Need to redo case_setup because we may have changed the number of threads case_setup(self._case, reset=True) From 5a2c68cf6f7fdb30127bf4ba74141cf874e9accc Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 6 Oct 2017 09:47:04 -0600 Subject: [PATCH 38/41] str conversion --- scripts/lib/CIME/SystemTests/system_tests_common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/lib/CIME/SystemTests/system_tests_common.py b/scripts/lib/CIME/SystemTests/system_tests_common.py index 4d5e9a52b9c..7f849c06c7b 100644 --- a/scripts/lib/CIME/SystemTests/system_tests_common.py +++ b/scripts/lib/CIME/SystemTests/system_tests_common.py @@ -157,7 +157,8 @@ def run(self): except BaseException as msg: success = False - if "RUN FAIL" in str(msg): + msg = str(msg) + if "RUN FAIL" in msg: # Don't want to print stacktrace for a model failure since that # is not a CIME/infrastructure problem. excmsg = msg From 8e51ac3804a54da2b9cd35ee310adb055965ddb8 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 10 Oct 2017 13:27:10 -0600 Subject: [PATCH 39/41] more exception handling fixes --- .../CIME/SystemTests/system_tests_common.py | 25 +++++++++++++------ scripts/lib/CIME/XML/env_base.py | 1 - scripts/lib/CIME/buildlib.py | 1 - scripts/tests/scripts_regression_tests.py | 4 +-- 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/scripts/lib/CIME/SystemTests/system_tests_common.py b/scripts/lib/CIME/SystemTests/system_tests_common.py index 7f849c06c7b..c8846c29215 100644 --- a/scripts/lib/CIME/SystemTests/system_tests_common.py +++ b/scripts/lib/CIME/SystemTests/system_tests_common.py @@ -88,16 +88,18 @@ def build(self, sharedlib_only=False, model_only=False): try: self.build_phase(sharedlib_only=(phase_name==SHAREDLIB_BUILD_PHASE), model_only=(phase_name==MODEL_BUILD_PHASE)) - except: + except BaseException as e: success = False - msg = sys.exc_info()[1].message - + try: + msg = e.message + except: + msg = str(e) if "BUILD FAIL" in msg: # Don't want to print stacktrace for a model failure since that # is not a CIME/infrastructure problem. excmsg = msg else: - excmsg = "Exception during build:\n{}\n{}".format(sys.exc_info()[1], traceback.format_exc()) + excmsg = "Exception during build:\n{}\n{}".format(msg, traceback.format_exc()) logger.warning(excmsg) append_testlog(excmsg) @@ -155,9 +157,12 @@ def run(self): self._check_for_memleak() - except BaseException as msg: + except BaseException as e: success = False - msg = str(msg) + try: + msg = e.message + except: + msg = str(e) if "RUN FAIL" in msg: # Don't want to print stacktrace for a model failure since that # is not a CIME/infrastructure problem. @@ -242,8 +247,12 @@ def _coupler_log_indicates_run_complete(self): try: if six.b("SUCCESSFUL TERMINATION") in gzip.open(cpllog, 'rb').read(): allgood = allgood - 1 - except BaseException as msg: - + except BaseException as e: + try: + msg = e.message + except: + msg = str(e) + logger.info("{} is not compressed, assuming run failed {}".format(cpllog, msg)) return allgood==0 diff --git a/scripts/lib/CIME/XML/env_base.py b/scripts/lib/CIME/XML/env_base.py index e5fa2e33830..3acfb36a3cd 100644 --- a/scripts/lib/CIME/XML/env_base.py +++ b/scripts/lib/CIME/XML/env_base.py @@ -1,7 +1,6 @@ """ Base class for env files. This class inherits from EntryID.py """ -import string from CIME.XML.standard_module_setup import * from CIME.XML.entry_id import EntryID from CIME.XML.headers import Headers diff --git a/scripts/lib/CIME/buildlib.py b/scripts/lib/CIME/buildlib.py index a2f97fc6a93..551f6325d75 100644 --- a/scripts/lib/CIME/buildlib.py +++ b/scripts/lib/CIME/buildlib.py @@ -6,7 +6,6 @@ from CIME.case import Case from CIME.utils import parse_args_and_handle_standard_logging_options, setup_standard_logging_options import sys, os, argparse, doctest -import six logger = logging.getLogger(__name__) ############################################################################### diff --git a/scripts/tests/scripts_regression_tests.py b/scripts/tests/scripts_regression_tests.py index de746573829..760a18774b7 100755 --- a/scripts/tests/scripts_regression_tests.py +++ b/scripts/tests/scripts_regression_tests.py @@ -2434,8 +2434,8 @@ def _main_func(): try: unittest.main(verbosity=2, catchbreak=True) - except SystemExit as had_fails: - if had_fails: + except SystemExit as e: + if e.__str__() != "False": print("Detected failures, leaving directory:", TEST_ROOT) else: print("All pass, removing directory:", TEST_ROOT) From 05c41b71621d2d83ad01769f0a6100c6ed8c9dce Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 10 Oct 2017 15:49:49 -0600 Subject: [PATCH 40/41] different way to handle exception msg --- .../lib/CIME/SystemTests/system_tests_common.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/scripts/lib/CIME/SystemTests/system_tests_common.py b/scripts/lib/CIME/SystemTests/system_tests_common.py index c8846c29215..a4e97d1b116 100644 --- a/scripts/lib/CIME/SystemTests/system_tests_common.py +++ b/scripts/lib/CIME/SystemTests/system_tests_common.py @@ -90,10 +90,7 @@ def build(self, sharedlib_only=False, model_only=False): model_only=(phase_name==MODEL_BUILD_PHASE)) except BaseException as e: success = False - try: - msg = e.message - except: - msg = str(e) + msg = e.__str__() if "BUILD FAIL" in msg: # Don't want to print stacktrace for a model failure since that # is not a CIME/infrastructure problem. @@ -159,10 +156,7 @@ def run(self): except BaseException as e: success = False - try: - msg = e.message - except: - msg = str(e) + msg = e.__str__() if "RUN FAIL" in msg: # Don't want to print stacktrace for a model failure since that # is not a CIME/infrastructure problem. @@ -248,10 +242,7 @@ def _coupler_log_indicates_run_complete(self): if six.b("SUCCESSFUL TERMINATION") in gzip.open(cpllog, 'rb').read(): allgood = allgood - 1 except BaseException as e: - try: - msg = e.message - except: - msg = str(e) + msg = e.__str__() logger.info("{} is not compressed, assuming run failed {}".format(cpllog, msg)) From b8837e899545db6f8bb64204f6cffc02aec6bfc4 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 12 Oct 2017 07:29:53 -0600 Subject: [PATCH 41/41] pylint and doctest fixes --- scripts/lib/CIME/namelist.py | 81 ++++++++++++++--------- scripts/lib/six.py | 16 ----- scripts/tests/scripts_regression_tests.py | 23 ++++--- 3 files changed, 61 insertions(+), 59 deletions(-) diff --git a/scripts/lib/CIME/namelist.py b/scripts/lib/CIME/namelist.py index acc608a262c..b4613d63ba1 100644 --- a/scripts/lib/CIME/namelist.py +++ b/scripts/lib/CIME/namelist.py @@ -641,7 +641,7 @@ def literal_to_python_value(literal, type_=None): >>> literal_to_python_value("") >>> literal_to_python_value("-1.D+10") -10000000000.0 - >>> six.shouldRaise(ValueError, literal_to_python_value, "nan(1234)") + >>> shouldRaise(ValueError, literal_to_python_value, "nan(1234)") """ expect(FORTRAN_REPEAT_PREFIX_REGEX.search(literal) is None, "Cannot use repetition syntax in literal_to_python_value") @@ -851,6 +851,23 @@ def parse(in_file=None, text=None, groupless=False, convert_tab_to_space=True): return Namelist(namelist_dict) +def shouldRaise(eclass, method, *args, **kw): + """ + A helper function to make doctests py3 compatible + http://python3porting.com/problems.html#running-doctests + """ + try: + method(*args, **kw) + except: + e = sys.exc_info()[1] + if not isinstance(e, eclass): + raise + return + raise Exception("Expected exception %s not raised" % + str(eclass)) + + + class Namelist(object): """Class representing a Fortran namelist. @@ -1275,7 +1292,7 @@ def _curr(self): def _next(self): """Return the character at the next position. - >>> six.shouldRaise(_NamelistEOF, _NamelistParser(' ')._next) + >>> shouldRaise(_NamelistEOF, _NamelistParser(' ')._next) """ # If at the end of the file, we should raise _NamelistEOF. The easiest @@ -1315,9 +1332,9 @@ def _advance(self, nchars=1, check_eof=False): >>> x._advance(3) >>> (x._pos, x._line, x._col) (7, 3, 1) - >>> six.shouldRaise(_NamelistEOF, x._advance, 1) + >>> shouldRaise(_NamelistEOF, x._advance, 1) - >>> six.shouldRaise(_NamelistEOF, _NamelistParser('abc\n')._advance, 4) + >>> shouldRaise(_NamelistEOF, _NamelistParser('abc\n')._advance, 4) >>> x = _NamelistParser('ab') >>> x._advance(check_eof=True) @@ -1361,7 +1378,7 @@ def _eat_whitespace(self, allow_initial_comment=False): >>> x._eat_whitespace() False >>> x._advance() - >>> six.shouldRaise(_NamelistEOF, x._eat_whitespace) + >>> shouldRaise(_NamelistEOF, x._eat_whitespace) >>> x = _NamelistParser(' \n! blah\n ! blah\n a') >>> x._eat_whitespace() @@ -1416,10 +1433,10 @@ def _eat_comment(self): >>> x._curr() 'a' >>> x._advance(2) - >>> six.shouldRaise(_NamelistEOF, x._eat_comment) + >>> shouldRaise(_NamelistEOF, x._eat_comment) >>> x = _NamelistParser('! foo\n') - >>> six.shouldRaise(_NamelistEOF, x._eat_comment) + >>> shouldRaise(_NamelistEOF, x._eat_comment) """ if self._curr() != '!': @@ -1444,7 +1461,7 @@ def _expect_char(self, chars): >>> x = _NamelistParser('ab') >>> x._expect_char('a') >>> x._advance() - >>> six.shouldRaise(_NamelistParseError, x._expect_char, 'a') + >>> shouldRaise(_NamelistParseError, x._expect_char, 'a') >>> x._expect_char('ab') """ @@ -1458,19 +1475,19 @@ def _expect_char(self, chars): def _parse_namelist_group_name(self): r"""Parses and returns a namelist group name at the current position. - >>> six.shouldRaise(_NamelistParseError, _NamelistParser('abc')._parse_namelist_group_name) + >>> shouldRaise(_NamelistParseError, _NamelistParser('abc')._parse_namelist_group_name) - >>> six.shouldRaise(_NamelistEOF, _NamelistParser('&abc')._parse_namelist_group_name) + >>> shouldRaise(_NamelistEOF, _NamelistParser('&abc')._parse_namelist_group_name) >>> _NamelistParser('&abc ')._parse_namelist_group_name() 'abc' >>> _NamelistParser('&abc\n')._parse_namelist_group_name() 'abc' - >>> six.shouldRaise(_NamelistParseError, _NamelistParser('&abc/ ')._parse_namelist_group_name) + >>> shouldRaise(_NamelistParseError, _NamelistParser('&abc/ ')._parse_namelist_group_name) - >>> six.shouldRaise(_NamelistParseError, _NamelistParser('&abc= ')._parse_namelist_group_name) + >>> shouldRaise(_NamelistParseError, _NamelistParser('&abc= ')._parse_namelist_group_name) - >>> six.shouldRaise(_NamelistParseError, _NamelistParser('& ')._parse_namelist_group_name) + >>> shouldRaise(_NamelistParseError, _NamelistParser('& ')._parse_namelist_group_name) """ self._expect_char("&") @@ -1484,7 +1501,7 @@ def _parse_variable_name(self, allow_equals=True): variable name; if it is `False`, only white space can be used for this purpose. - >>> six.shouldRaise(_NamelistEOF, _NamelistParser('abc')._parse_variable_name) + >>> shouldRaise(_NamelistEOF, _NamelistParser('abc')._parse_variable_name) >>> _NamelistParser('foo(2)= ')._parse_variable_name() 'foo(2)' @@ -1556,13 +1573,13 @@ def _parse_character_literal(self): Position on return is the last character of the string; we avoid advancing past that in order to avoid potential EOF errors. - >>> six.shouldRaise(_NamelistEOF, _NamelistParser('"abc')._parse_character_literal) + >>> shouldRaise(_NamelistEOF, _NamelistParser('"abc')._parse_character_literal) >>> _NamelistParser('"abc" ')._parse_character_literal() '"abc"' >>> _NamelistParser("'abc' ")._parse_character_literal() "'abc'" - >>> six.shouldRaise(_NamelistParseError, _NamelistParser("*abc* ")._parse_character_literal) + >>> shouldRaise(_NamelistParseError, _NamelistParser("*abc* ")._parse_character_literal) >>> _NamelistParser("'abc''def' ")._parse_character_literal() "'abc''def'" @@ -1596,11 +1613,11 @@ def _parse_complex_literal(self): Position on return is the last character of the string; we avoid advancing past that in order to avoid potential EOF errors. - >>> six.shouldRaise(_NamelistEOF, _NamelistParser('(1.,2.')._parse_complex_literal) + >>> shouldRaise(_NamelistEOF, _NamelistParser('(1.,2.')._parse_complex_literal) >>> _NamelistParser('(1.,2.) ')._parse_complex_literal() '(1.,2.)' - >>> six.shouldRaise(_NamelistParseError, _NamelistParser("(A,B) ")._parse_complex_literal) + >>> shouldRaise(_NamelistParseError, _NamelistParser("(A,B) ")._parse_complex_literal) """ old_pos = self._pos @@ -1680,13 +1697,13 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): '"abc"' >>> _NamelistParser("'abc' ")._parse_literal() "'abc'" - >>> six.shouldRaise(_NamelistEOF, _NamelistParser('"abc"')._parse_literal) + >>> shouldRaise(_NamelistEOF, _NamelistParser('"abc"')._parse_literal) >>> _NamelistParser('"abc"')._parse_literal(allow_eof_end=True) '"abc"' >>> _NamelistParser('(1.,2.) ')._parse_literal() '(1.,2.)' - >>> six.shouldRaise(_NamelistEOF, _NamelistParser('(1.,2.)')._parse_literal) + >>> shouldRaise(_NamelistEOF, _NamelistParser('(1.,2.)')._parse_literal) >>> _NamelistParser('(1.,2.)')._parse_literal(allow_eof_end=True) '(1.,2.)' @@ -1700,7 +1717,7 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): 'nan(booga)' >>> _NamelistParser('.FLORIDA$ ')._parse_literal() '.FLORIDA$' - >>> six.shouldRaise(_NamelistParseError, _NamelistParser('hamburger ')._parse_literal) + >>> shouldRaise(_NamelistParseError, _NamelistParser('hamburger ')._parse_literal) >>> _NamelistParser('5,')._parse_literal() '5' @@ -1716,13 +1733,13 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): '6*(1., 2.)' >>> _NamelistParser('6*"a" ')._parse_literal() '6*"a"' - >>> six.shouldRaise(_NamelistEOF, _NamelistParser('6*')._parse_literal) + >>> shouldRaise(_NamelistEOF, _NamelistParser('6*')._parse_literal) >>> _NamelistParser('6*')._parse_literal(allow_eof_end=True) '6*' - >>> six.shouldRaise(_NamelistParseError, _NamelistParser('foo= ')._parse_literal) + >>> shouldRaise(_NamelistParseError, _NamelistParser('foo= ')._parse_literal) - >>> six.shouldRaise(_NamelistParseError, _NamelistParser('foo+= ')._parse_literal) + >>> shouldRaise(_NamelistParseError, _NamelistParser('foo+= ')._parse_literal) >>> _NamelistParser('5,')._parse_literal(allow_name=True) '5' @@ -1734,9 +1751,9 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): >>> x._parse_literal(allow_name=True) >>> x._curr() 'f' - >>> six.shouldRaise(_NamelistParseError, _NamelistParser('6*foo= ')._parse_literal, allow_name=True) + >>> shouldRaise(_NamelistParseError, _NamelistParser('6*foo= ')._parse_literal, allow_name=True) - >>> six.shouldRaise(_NamelistParseError, _NamelistParser('6*foo+= ')._parse_literal, allow_name=True) + >>> shouldRaise(_NamelistParseError, _NamelistParser('6*foo+= ')._parse_literal, allow_name=True) >>> x = _NamelistParser('foo = ') >>> x._parse_literal(allow_name=True) @@ -1839,7 +1856,7 @@ def _expect_separator(self, allow_eof=False): >>> x._curr() '/' >>> x = _NamelistParser("a") - >>> six.shouldRaise(_NamelistParseError, x._expect_separator) + >>> shouldRaise(_NamelistParseError, x._expect_separator) >>> x = _NamelistParser(" , a") >>> x._expect_separator() @@ -1870,7 +1887,7 @@ def _expect_separator(self, allow_eof=False): >>> x._expect_separator(allow_eof=True) True >>> x = _NamelistParser(" / ") - >>> six.shouldRaise(_NamelistParseError, x._expect_separator, allow_eof=True) + >>> shouldRaise(_NamelistParseError, x._expect_separator, allow_eof=True) """ errstring = "found group-terminating '/' in file without group names" @@ -1920,7 +1937,7 @@ def _parse_name_and_values(self, allow_eof_end=False): ('foo', ["'bar'"], False) >>> _NamelistParser("foo=\n'bar' /")._parse_name_and_values() ('foo', ["'bar'"], False) - >>> six.shouldRaise(_NamelistParseError, _NamelistParser("foo 'bar' /")._parse_name_and_values) + >>> shouldRaise(_NamelistParseError, _NamelistParser("foo 'bar' /")._parse_name_and_values) >>> _NamelistParser("foo='bar','bazz' /")._parse_name_and_values() ('foo', ["'bar'", "'bazz'"], False) @@ -1930,13 +1947,13 @@ def _parse_name_and_values(self, allow_eof_end=False): ('foo', ["'bar'", "'bazz'"], False) >>> _NamelistParser("foo='bar' 'bazz' foo2(2)='ban'")._parse_name_and_values() ('foo', ["'bar'", "'bazz'"], False) - >>> six.shouldRaise(_NamelistParseError, _NamelistParser("foo= foo2='ban' ")._parse_name_and_values) + >>> shouldRaise(_NamelistParseError, _NamelistParser("foo= foo2='ban' ")._parse_name_and_values) >>> _NamelistParser("foo=,,'bazz',6* ")._parse_name_and_values(allow_eof_end=True) ('foo', ['', '', "'bazz'", '6*'], False) >>> _NamelistParser("foo(3)='bazz'")._parse_name_and_values(allow_eof_end=True) ('foo(3)', ["'bazz'"], False) - >>> six.shouldRaise(_NamelistEOF, _NamelistParser("foo=")._parse_name_and_values) + >>> shouldRaise(_NamelistEOF, _NamelistParser("foo=")._parse_name_and_values) >>> _NamelistParser("foo=")._parse_name_and_values(allow_eof_end=True) ('foo', [''], False) @@ -2016,7 +2033,7 @@ def _parse_namelist_group(self): >>> x = _NamelistParser("&group /&group /") >>> x._parse_namelist_group() >>> x._advance() - >>> six.shouldRaise(_NamelistParseError, x._parse_namelist_group) + >>> shouldRaise(_NamelistParseError, x._parse_namelist_group) >>> x = _NamelistParser("&group foo='bar', foo='bazz' /") >>> x._parse_namelist_group() diff --git a/scripts/lib/six.py b/scripts/lib/six.py index 5e5ae0eef87..a0297d7113d 100644 --- a/scripts/lib/six.py +++ b/scripts/lib/six.py @@ -865,22 +865,6 @@ def python_2_unicode_compatible(klass): klass.__str__ = lambda self: self.__unicode__().encode('utf-8') return klass - -def shouldRaise(eclass, method, *args, **kw): - """ - http://python3porting.com/problems.html#running-doctests - """ - try: - method(*args, **kw) - except: - e = sys.exc_info()[1] - if not isinstance(e, eclass): - raise - return - raise Exception("Expected exception %s not raised" % - str(eclass)) - - # Complete the moves implementation. # This code is at the end of this module to speed up module loading. # Turn this module into a package. diff --git a/scripts/tests/scripts_regression_tests.py b/scripts/tests/scripts_regression_tests.py index 760a18774b7..4742e1c2b91 100755 --- a/scripts/tests/scripts_regression_tests.py +++ b/scripts/tests/scripts_regression_tests.py @@ -10,6 +10,7 @@ # Remove all pyc files to ensure we're testing the right things import subprocess subprocess.call('/bin/rm $(find . -name "*.pyc")', shell=True, cwd=LIB_DIR) +from six import assertRaisesRegex import six from CIME.utils import run_cmd, run_cmd_no_fail, get_lids, get_current_commit @@ -1939,7 +1940,7 @@ def test_script_rejects_bad_build_system(self): """The macro writer rejects a bad build system string.""" maker = Compilers(MockMachines("mymachine", "SomeOS"), version=2.0) bad_string = "argle-bargle." - with six.assertRaisesRegex(self, + with assertRaisesRegex(self, SystemExit, "Unrecognized build system provided to write_macros: " + bad_string): get_macros(maker, "This string is irrelevant.", bad_string) @@ -2041,7 +2042,7 @@ def test_reject_duplicate_defaults(self): """The macro writer dies if given many defaults.""" xml1 = """/path/to/default""" xml2 = """/path/to/other_default""" - with six.assertRaisesRegex(self, + with assertRaisesRegex(self, SystemExit, "Variable MPI_PATH is set ambiguously in config_build.xml."): self.xml_to_tester(xml1+xml2) @@ -2050,7 +2051,7 @@ def test_reject_duplicates(self): """The macro writer dies if given many matches for a given configuration.""" xml1 = """/path/to/mpich""" xml2 = """/path/to/mpich2""" - with six.assertRaisesRegex(self, + with assertRaisesRegex(self, SystemExit, "Variable MPI_PATH is set ambiguously in config_build.xml."): self.xml_to_tester(xml1+xml2) @@ -2059,7 +2060,7 @@ def test_reject_ambiguous(self): """The macro writer dies if given an ambiguous set of matches.""" xml1 = """/path/to/mpich""" xml2 = """/path/to/mpi-debug""" - with six.assertRaisesRegex(self, + with assertRaisesRegex(self, SystemExit, "Variable MPI_PATH is set ambiguously in config_build.xml."): self.xml_to_tester(xml1+xml2) @@ -2191,7 +2192,7 @@ def test_env_and_shell_command(self): tester.assert_variable_equals("FFLAGS", "-O2 -fast", env={"OPT_LEVEL": "2"}) xml1 = """-O$SHELL{echo $ENV{OPT_LEVEL}} -fast""" err_msg = "Nesting not allowed.*" - with six.assertRaisesRegex(self,SystemExit, err_msg): + with assertRaisesRegex(self,SystemExit, err_msg): self.xml_to_tester(xml1) def test_config_variable_insertion(self): @@ -2221,12 +2222,12 @@ def test_config_reject_self_references(self): # references. xml1 = """MPI_LIB_NAME""" err_msg = ".* has bad references." - with six.assertRaisesRegex(self,SystemExit, err_msg): + with assertRaisesRegex(self,SystemExit, err_msg): self.xml_to_tester(""+xml1+"") xml1 = """${MPI_LIB_NAME}""" err_msg = ".* has bad references." - with six.assertRaisesRegex(self,SystemExit, err_msg): + with assertRaisesRegex(self,SystemExit, err_msg): self.xml_to_tester(""+xml1+"") def test_config_reject_cyclical_references(self): @@ -2234,12 +2235,12 @@ def test_config_reject_cyclical_references(self): xml1 = """MPI_PATH""" xml2 = """MPI_LIB_NAME""" err_msg = ".* has bad references." - with six.assertRaisesRegex(self,SystemExit, err_msg): + with assertRaisesRegex(self,SystemExit, err_msg): self.xml_to_tester(""+xml1+xml2+"") xml1 = """${MPI_PATH}""" xml2 = """${MPI_LIB_NAME}""" err_msg = ".* has bad references." - with six.assertRaisesRegex(self,SystemExit, err_msg): + with assertRaisesRegex(self,SystemExit, err_msg): self.xml_to_tester(""+xml1+xml2+"") def test_variable_insertion_with_machine_specific_setting(self): @@ -2248,14 +2249,14 @@ def test_variable_insertion_with_machine_specific_setting(self): xml2 = """MPI_PATH""".format(self.test_machine) xml3 = """MPI_LIB_NAME""" err_msg = ".* has bad references." - with six.assertRaisesRegex(self,SystemExit, err_msg): + with assertRaisesRegex(self,SystemExit, err_msg): self.xml_to_tester(xml1+xml2+xml3) xml1 = """something""" xml2 = """MPI_PATH""".format(self.test_machine) xml3 = """${MPI_LIB_NAME}""" err_msg = ".* has bad references." - with six.assertRaisesRegex(self,SystemExit, err_msg): + with assertRaisesRegex(self,SystemExit, err_msg): self.xml_to_tester(xml1+xml2+xml3)