From 97244cbe5769dc5b92cb8092d95c8fd501d1bbbd Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 17 Mar 2017 08:31:54 -0600 Subject: [PATCH 01/25] add _nameindex to NamelistParser --- scripts/lib/CIME/namelist.py | 99 +++++++++++++++++++++++++++++++----- scripts/lib/CIME/nmlgen.py | 9 ++-- 2 files changed, 92 insertions(+), 16 deletions(-) diff --git a/scripts/lib/CIME/namelist.py b/scripts/lib/CIME/namelist.py index f2f02cf752a..6900a69a5bf 100644 --- a/scripts/lib/CIME/namelist.py +++ b/scripts/lib/CIME/namelist.py @@ -112,7 +112,7 @@ # Fortran syntax regular expressions. # Variable names. -FORTRAN_NAME_REGEX = re.compile(r"^[a-z][a-z0-9_]{0,62}$", re.IGNORECASE) +FORTRAN_NAME_REGEX = re.compile(r"^[a-z][a-z0-9_]{0,62}(\((\d+)\))?$", re.IGNORECASE) FORTRAN_LITERAL_REGEXES = {} # Integer literals. _int_re_string = r"(\+|-)?[0-9]+" @@ -171,6 +171,20 @@ def is_valid_fortran_name(string): """ return FORTRAN_NAME_REGEX.search(string) is not None +def fortran_name_index(string): + """ Get name index if provided + + >>> fortran_name_index("foo(2)") + 2 + >>> fortran_name_index("bar") + + >>> fortran_name_index("foo(a)") + + """ + m = FORTRAN_NAME_REGEX.search(string) + if m is not None and m.group(2) is not None: + return int(m.group(2)) + def fortran_namelist_base_value(string): r"""Strip off whitespace and repetition syntax from a namelist value. @@ -601,7 +615,7 @@ def literal_to_python_value(literal, type_=None): return float(literal) -def expand_literal_list(literals): +def expand_literal_list(literals, nindex=None): """Expands a list of literal values to get rid of repetition syntax. >>> expand_literal_list([]) @@ -610,6 +624,8 @@ def expand_literal_list(literals): ['true'] >>> expand_literal_list(['1', '2', 'f*', '3*3', '5']) ['1', '2', 'f*', '3', '3', '3', '5'] + >>> expand_literal_list(['1'],nindex=2) + ['', '1'] >>> expand_literal_list([u'2*f*']) [u'f*', u'f*'] """ @@ -618,6 +634,10 @@ def expand_literal_list(literals): if FORTRAN_REPEAT_PREFIX_REGEX.search(literal) is not None: num, _, value = literal.partition('*') expanded += int(num) * [value] + elif nindex is not None: + for i in range(0,nindex-1): + expanded.append('') + expanded.append(literal) else: expanded.append(literal) return expanded @@ -673,7 +693,7 @@ def compress_literal_list(literals): compressed.append(str(literal)) return compressed -def merge_literal_lists(default, overwrite): +def merge_literal_lists(default, overwrite, dindex=None, oindex=None): """Merge two lists of literal value strings. The `overwrite` values have higher precedence, so will overwrite the @@ -695,12 +715,22 @@ def merge_literal_lists(default, overwrite): ['true'] >>> merge_literal_lists(['true'], []) ['true'] + >>> merge_literal_lists(['true'], ['false'], dindex=2) + ['false', 'true'] + >>> merge_literal_lists(['true'], [], dindex=2) + ['', 'true'] + >>> merge_literal_lists([], ['foo'], oindex=2) + ['', 'foo'] + >>> merge_literal_lists(['true','true','true'], ['false'], oindex=2) + ['true', 'false', 'true'] + >>> merge_literal_lists(['false'],['true','true','true'], dindex=2) + ['true', 'true', 'true'] >>> merge_literal_lists(['3*false', '3*true'], ['true', '4*', 'false']) ['true', 'false', 'false', 'true', 'true', 'false'] """ merged = [] - default = expand_literal_list(default) - overwrite = expand_literal_list(overwrite) + default = expand_literal_list(default, dindex) + overwrite = expand_literal_list(overwrite, oindex) for default_elem, elem in zip(default, overwrite): if elem == '': merged.append(default_elem) @@ -870,6 +900,8 @@ def get_value(self, variable_name): SystemExit: ERROR: Namelist.get_value: Variable %s is present in multiple groups: [u'bazz', u'foo'] >>> parse(text='&foo bar=1 / &bazz /').get_value('Bar') [u'1'] + >>> parse(text='&foo bar(2)=1 / &bazz /').get_value('Bar') + [u'1'] >>> parse(text='&foo / &bazz /').get_value('bar') [u''] """ @@ -1029,7 +1061,7 @@ def _write(self, out_file, groups, format_, sorted_groups): values = group[name] # @ is used in a namelist to put the same namelist variable in multiple groups - # in the write phase, all characters in the namelist variable name after + # in the write phase, all characters in the namelist variable name after # the @ and including the @ should be removed if "@" in name: name = re.sub('@.+$', "", name) @@ -1121,6 +1153,10 @@ def __init__(self, text, groupless=False): # name-value pairs as values. (Or a single flat dictionary if # `groupless=True`.) self._settings = {} + # Fortran allows setting a particular index of an array + # such as foo(2)='k' + # this dict is set to that value if used. + self._nameindex = {} self._groupless = groupless def _line_col_string(self): @@ -1379,6 +1415,8 @@ def _parse_variable_name(self, allow_equals=True): Traceback (most recent call last): ... _NamelistEOF: Unexpected end of file encountered in namelist. (At line 1, column 3) + >>> _NamelistParser('foo(2)= ')._parse_variable_name() + u'foo' >>> _NamelistParser('abc ')._parse_variable_name() u'abc' >>> _NamelistParser('ABC ')._parse_variable_name() @@ -1401,9 +1439,8 @@ def _parse_variable_name(self, allow_equals=True): while self._curr() not in separators: self._advance() text = self._text[old_pos:self._pos] - # @ is used in a namelist to put the same namelist variable in multiple groups - # in the write phase, all characters in the namelist variable name after + # in the write phase, all characters in the namelist variable name after # the @ and including the @ should be removed if "%" in text: text_check = re.sub('%.+$', "", text) @@ -1414,6 +1451,18 @@ def _parse_variable_name(self, allow_equals=True): if not is_valid_fortran_name(text_check): raise _NamelistParseError("%r is not a valid variable name at %s" % (str(text), self._line_col_string())) + + # a particular value of a namelist array may be assigned using syntax + # foo(2) = 7 + + nindex = fortran_name_index(text_check) + if nindex is not None: + text = text_check[:text.find('(')] + vname = text.lower() + if vname in self._nameindex: + self._nameindex[vname].append(nindex) + else: + self._nameindex[vname] = [nindex] return text.lower() def _parse_character_literal(self): @@ -1766,6 +1815,8 @@ def _parse_name_and_values(self, allow_eof_end=False): >>> _NamelistParser("foo='bar' /")._parse_name_and_values() (u'foo', [u"'bar'"]) + >>> _NamelistParser("foo(3)='bar' /")._parse_name_and_values() + (u'foo', [u"'bar'"]) >>> _NamelistParser("foo ='bar' /")._parse_name_and_values() (u'foo', [u"'bar'"]) >>> _NamelistParser("foo=\n'bar' /")._parse_name_and_values() @@ -1780,12 +1831,16 @@ def _parse_name_and_values(self, allow_eof_end=False): (u'foo', [u'', u'', u"'bazz'", u'6*']) >>> _NamelistParser("foo='bar' 'bazz' foo2='ban'")._parse_name_and_values() (u'foo', [u"'bar'", u"'bazz'"]) + >>> _NamelistParser("foo='bar' 'bazz' foo2(2)='ban'")._parse_name_and_values() + (u'foo', [u"'bar'", u"'bazz'"]) >>> _NamelistParser("foo= foo2='ban' ")._parse_name_and_values() Traceback (most recent call last): ... _NamelistParseError: Error in parsing namelist: expected literal value, but got "foo2='ban'" at line 1, column 15 >>> _NamelistParser("foo=,,'bazz',6* ")._parse_name_and_values(allow_eof_end=True) (u'foo', [u'', u'', u"'bazz'", u'6*']) + >>> _NamelistParser("foo(3)='bazz'")._parse_name_and_values(allow_eof_end=True) + (u'foo', [u"'bazz'"]) >>> _NamelistParser("foo=")._parse_name_and_values() Traceback (most recent call last): ... @@ -1878,14 +1933,20 @@ def _parse_namelist_group(self): self._eat_whitespace() while self._curr() != '/': name, values = self._parse_name_and_values() + oindex = None + dsettings = [] + if name in self._nameindex: + oindex = self._nameindex[name].pop(0) if self._groupless: if name in self._settings: - values = merge_literal_lists(self._settings[name], values) + dsettings = self._settings[name] + values = merge_literal_lists(dsettings, values, oindex=oindex) self._settings[name] = values else: group = self._settings[group_name] if name in group: - values = merge_literal_lists(group[name], values) + dsettings = group[name] + values = merge_literal_lists(dsettings, values, oindex=oindex) group[name] = values def parse_namelist(self): @@ -1907,15 +1968,21 @@ def parse_namelist(self): >>> _NamelistParser("&group1\n foo='bar','bazz'\n,, foo2=2*5\n / &group2 /").parse_namelist() {u'group1': {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'2*5']}, u'group2': {}} >>> _NamelistParser("!blah \n foo='bar','bazz'\n,, foo2=2*5\n ", groupless=True).parse_namelist() - {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'2*5']} + {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'5', u'5']} >>> _NamelistParser("!blah \n foo='bar','bazz'\n,, foo2=2*5,6\n ", groupless=True).parse_namelist() - {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'2*5', u'6']} + {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'5', u'5', u'6']} >>> _NamelistParser("!blah \n foo='bar'", groupless=True).parse_namelist() {u'foo': [u"'bar'"]} >>> _NamelistParser("foo='bar', foo='bazz'", groupless=True).parse_namelist() {u'foo': [u"'bazz'"]} >>> _NamelistParser("foo='bar', foo=", groupless=True).parse_namelist() {u'foo': [u"'bar'"]} + >>> _NamelistParser("foo='bar', foo(3)='bazz'", groupless=True).parse_namelist() + {u'foo': [u"'bar'", '', u"'bazz'"]} + >>> _NamelistParser("foo(2)='bar'", groupless=True).parse_namelist() + {u'foo': ['', u"'bar'"]} + >>> _NamelistParser("foo(2)='bar', foo(3)='bazz'", groupless=True).parse_namelist() + {u'foo': ['', u"'bar'", u"'bazz'"]} """ # Return empty dictionary for empty files. if self._len == 0: @@ -1930,8 +1997,14 @@ def parse_namelist(self): if self._groupless and self._curr() != '&': while self._pos < self._len: name, values = self._parse_name_and_values(allow_eof_end=True) + oindex = None + if name in self._nameindex: + oindex = self._nameindex[name].pop(0) + dsettings = [] if name in self._settings: - values = merge_literal_lists(self._settings[name], values) + dsettings = self._settings[name] + values = merge_literal_lists(dsettings, values, + oindex=oindex) self._settings[name] = values return self._settings # Loop over namelist groups in the file. diff --git a/scripts/lib/CIME/nmlgen.py b/scripts/lib/CIME/nmlgen.py index 0b41c35effb..aaf300f2668 100644 --- a/scripts/lib/CIME/nmlgen.py +++ b/scripts/lib/CIME/nmlgen.py @@ -274,6 +274,7 @@ def get_default(self, name, config=None, allow_none=False): default[i] = self.quote_string(scalar) default = self._to_python_value(name, default) + return default def get_streams(self): @@ -407,6 +408,7 @@ def create_stream_file_and_update_shr_strdata_nml(self, config, #pylint:disable= `stream_path` - Path to write the stream file to. `data_list_path` - Path of file to append input data information to. """ + # Stream-specific configuration. config = config.copy() config["stream"] = stream @@ -520,10 +522,11 @@ def add_default(self, name, value=None, ignore_abs_path=None): # Check for existing value. current_literals = self._namelist.get_variable_value(group, name) - if current_literals != [""]: - have_value = True +# I may have broken something here??? +# if current_literals != [""]: +# have_value = True # Do not proceed further since this has been obtained the -infile contents - return +# return # Check for input argument. if value is not None: From ae9606729d3ed191baf4bd7235b827ff0c99f11b Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 17 Mar 2017 08:50:29 -0600 Subject: [PATCH 02/25] fix doctest issues --- scripts/lib/CIME/namelist.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/lib/CIME/namelist.py b/scripts/lib/CIME/namelist.py index 6900a69a5bf..b51bc838522 100644 --- a/scripts/lib/CIME/namelist.py +++ b/scripts/lib/CIME/namelist.py @@ -901,7 +901,7 @@ def get_value(self, variable_name): >>> parse(text='&foo bar=1 / &bazz /').get_value('Bar') [u'1'] >>> parse(text='&foo bar(2)=1 / &bazz /').get_value('Bar') - [u'1'] + ['', u'1'] >>> parse(text='&foo / &bazz /').get_value('bar') [u''] """ @@ -1896,11 +1896,11 @@ def _parse_namelist_group(self): >>> x = _NamelistParser("&group\n foo='bar','bazz'\n,, foo2=2*5\n /") >>> x._parse_namelist_group() >>> x._settings - {u'group': {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'2*5']}} + {u'group': {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'5', u'5']}} >>> x = _NamelistParser("&group\n foo='bar','bazz'\n,, foo2=2*5\n /", groupless=True) >>> x._parse_namelist_group() >>> x._settings - {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'2*5']} + {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'5', u'5']} >>> x._curr() u'/' >>> x = _NamelistParser("&group /&group /") @@ -1966,7 +1966,7 @@ def parse_namelist(self): >>> _NamelistParser("! Comment \n &group /! Comment ").parse_namelist() {u'group': {}} >>> _NamelistParser("&group1\n foo='bar','bazz'\n,, foo2=2*5\n / &group2 /").parse_namelist() - {u'group1': {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'2*5']}, u'group2': {}} + {u'group1': {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'5', u'5']}, u'group2': {}} >>> _NamelistParser("!blah \n foo='bar','bazz'\n,, foo2=2*5\n ", groupless=True).parse_namelist() {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'5', u'5']} >>> _NamelistParser("!blah \n foo='bar','bazz'\n,, foo2=2*5,6\n ", groupless=True).parse_namelist() From 71051845cb0fa83fcea085444b797e0b971e438f Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 17 Mar 2017 10:28:29 -0600 Subject: [PATCH 03/25] better assignment --- scripts/lib/CIME/namelist.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/lib/CIME/namelist.py b/scripts/lib/CIME/namelist.py index b51bc838522..134498881eb 100644 --- a/scripts/lib/CIME/namelist.py +++ b/scripts/lib/CIME/namelist.py @@ -635,8 +635,7 @@ def expand_literal_list(literals, nindex=None): num, _, value = literal.partition('*') expanded += int(num) * [value] elif nindex is not None: - for i in range(0,nindex-1): - expanded.append('') + expanded = ['']*(nindex-1) expanded.append(literal) else: expanded.append(literal) From 388f7d2127a25d3dc65a7645e2728ea81dedefdd Mon Sep 17 00:00:00 2001 From: Mariana Vertenstein Date: Fri, 17 Mar 2017 13:18:58 -0600 Subject: [PATCH 04/25] fixed multiple instance problem --- scripts/lib/CIME/nmlgen.py | 6 +++++- src/components/data_comps/dlnd/cime_config/buildnml | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/lib/CIME/nmlgen.py b/scripts/lib/CIME/nmlgen.py index aaf300f2668..b0cf38d3022 100644 --- a/scripts/lib/CIME/nmlgen.py +++ b/scripts/lib/CIME/nmlgen.py @@ -85,7 +85,7 @@ def __init__(self, case, definition_files, files=None): # Create namelist object. self._namelist = Namelist() - + # Define __enter__ and __exit__ so that we can use this as a context manager def __enter__(self): return self @@ -281,6 +281,10 @@ def get_streams(self): """Get a list of all streams used for the current data model mode.""" return self.get_default("streamslist") + def clean_streams(self): + for variable in self._streams_variables: + self._streams_namelists[variable] = [] + def _sub_fields(self, varnames): """Substitute indicators with given values in a list of fields. diff --git a/src/components/data_comps/dlnd/cime_config/buildnml b/src/components/data_comps/dlnd/cime_config/buildnml index 0d3298b483a..a3cc548a0b8 100755 --- a/src/components/data_comps/dlnd/cime_config/buildnml +++ b/src/components/data_comps/dlnd/cime_config/buildnml @@ -90,6 +90,7 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen): # shr_strdata_nml group and input data list. #---------------------------------------------------- for stream in streams: + nmlgen.clean_streams() # Ignore null values. if stream is None or stream in ("NULL", ""): @@ -108,7 +109,6 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen): nmlgen.update_shr_strdata_nml(config, stream, stream_path) else: nmlgen.create_stream_file_and_update_shr_strdata_nml(config, stream, stream_path, data_list_path) - #---------------------------------------------------- # Create `shr_strdata_nml` namelist group. #---------------------------------------------------- From 9be48ca45fc1908190a339ea0594f7be11069794 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 17 Mar 2017 14:39:03 -0600 Subject: [PATCH 05/25] need to clean streams between instances --- scripts/lib/CIME/buildnml.py | 1 - scripts/lib/CIME/namelist.py | 3 +++ scripts/lib/CIME/nmlgen.py | 9 ++++++++- src/components/data_comps/dlnd/cime_config/buildnml | 6 +----- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/scripts/lib/CIME/buildnml.py b/scripts/lib/CIME/buildnml.py index 2d90636a7b1..34f863155b6 100644 --- a/scripts/lib/CIME/buildnml.py +++ b/scripts/lib/CIME/buildnml.py @@ -118,6 +118,5 @@ def create_namelist_infile(case, user_nl_file, namelist_infile, infile_text=""): lines_output.append(line) lines_output.append("/ \n") - with open(namelist_infile, "w") as file_infile: file_infile.write("\n".join(lines_output)) diff --git a/scripts/lib/CIME/namelist.py b/scripts/lib/CIME/namelist.py index 134498881eb..381927b6218 100644 --- a/scripts/lib/CIME/namelist.py +++ b/scripts/lib/CIME/namelist.py @@ -839,6 +839,9 @@ def __init__(self, groups=None): self._groups[group_lc][variable_lc] = \ groups[group_name][variable_name] + def clean_groups(self): + self._groups = {} + def get_group_names(self): """Return a list of all groups in the namelist. diff --git a/scripts/lib/CIME/nmlgen.py b/scripts/lib/CIME/nmlgen.py index b0cf38d3022..14aaaa74f02 100644 --- a/scripts/lib/CIME/nmlgen.py +++ b/scripts/lib/CIME/nmlgen.py @@ -85,7 +85,7 @@ def __init__(self, case, definition_files, files=None): # Create namelist object. self._namelist = Namelist() - + # Define __enter__ and __exit__ so that we can use this as a context manager def __enter__(self): return self @@ -284,6 +284,13 @@ def get_streams(self): def clean_streams(self): for variable in self._streams_variables: self._streams_namelists[variable] = [] + self._streams_namelists["streams"] = [] + + def new_instance(self): + """ Clean the object just enough to introduce a new instance """ + self.clean_streams() + self._namelist.clean_groups() + def _sub_fields(self, varnames): """Substitute indicators with given values in a list of fields. diff --git a/src/components/data_comps/dlnd/cime_config/buildnml b/src/components/data_comps/dlnd/cime_config/buildnml index a3cc548a0b8..ae83c473826 100755 --- a/src/components/data_comps/dlnd/cime_config/buildnml +++ b/src/components/data_comps/dlnd/cime_config/buildnml @@ -84,14 +84,11 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen): # Construct the list of streams. #---------------------------------------------------- streams = nmlgen.get_streams() - #---------------------------------------------------- # For each stream, create stream text file and update # shr_strdata_nml group and input data list. #---------------------------------------------------- for stream in streams: - nmlgen.clean_streams() - # Ignore null values. if stream is None or stream in ("NULL", ""): continue @@ -162,12 +159,11 @@ def buildnml(case, caseroot, compname): # Create the namelist generator object - independent of instance nmlgen = NamelistGenerator(case, definition_file, files=files) - #---------------------------------------------------- # Loop over instances #---------------------------------------------------- for inst_counter in range(1, ninst+1): - + nmlgen.new_instance() # determine instance string inst_string = "" if ninst > 1: From 91b899a60cda48aaccc17afa1c827148b02981d6 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 17 Mar 2017 15:02:17 -0600 Subject: [PATCH 06/25] add new_instance function to all data models --- src/components/data_comps/datm/cime_config/buildnml | 2 +- src/components/data_comps/desp/cime_config/buildnml | 8 ++++---- src/components/data_comps/dice/cime_config/buildnml | 2 +- src/components/data_comps/docn/cime_config/buildnml | 2 +- src/components/data_comps/drof/cime_config/buildnml | 2 +- src/components/data_comps/dwav/cime_config/buildnml | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/components/data_comps/datm/cime_config/buildnml b/src/components/data_comps/datm/cime_config/buildnml index 1ed9fda2b9c..3e2fa726d37 100755 --- a/src/components/data_comps/datm/cime_config/buildnml +++ b/src/components/data_comps/datm/cime_config/buildnml @@ -203,7 +203,7 @@ def buildnml(case, caseroot, compname): # Loop over instances #---------------------------------------------------- for inst_counter in range(1, ninst+1): - + nmlgen.new_instance() # determine instance string inst_string = "" if ninst > 1: diff --git a/src/components/data_comps/desp/cime_config/buildnml b/src/components/data_comps/desp/cime_config/buildnml index 251b3cfa2c8..d097974adda 100755 --- a/src/components/data_comps/desp/cime_config/buildnml +++ b/src/components/data_comps/desp/cime_config/buildnml @@ -65,7 +65,7 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen): config['desp_mode'] = desp_mode #---------------------------------------------------- - # Initialize namelist defaults + # Initialize namelist defaults #---------------------------------------------------- nmlgen.init_defaults(infile, config) @@ -99,7 +99,7 @@ def buildnml(case, caseroot, compname): os.makedirs(confdir) #---------------------------------------------------- - # Construct the namelist generator + # Construct the namelist generator #---------------------------------------------------- # Determine directory for user modified namelist_definitions.xml and namelist_defaults.xml user_xml_dir = os.path.join(caseroot, "SourceMods", "src." + compname) @@ -117,12 +117,12 @@ def buildnml(case, caseroot, compname): # Create the namelist generator object - independent of instance nmlgen = NamelistGenerator(case, definition_file) - + #---------------------------------------------------- # Loop over instances #---------------------------------------------------- for inst_counter in range(1, ninst+1): - + nmlgen.new_instance() # determine instance string inst_string = "" if ninst > 1: diff --git a/src/components/data_comps/dice/cime_config/buildnml b/src/components/data_comps/dice/cime_config/buildnml index 45f18b0960c..341286f0d4e 100755 --- a/src/components/data_comps/dice/cime_config/buildnml +++ b/src/components/data_comps/dice/cime_config/buildnml @@ -164,7 +164,7 @@ def buildnml(case, caseroot, compname): # Loop over instances #---------------------------------------------------- for inst_counter in range(1, ninst+1): - + nmlgen.new_instance() # determine instance string inst_string = "" if ninst > 1: diff --git a/src/components/data_comps/docn/cime_config/buildnml b/src/components/data_comps/docn/cime_config/buildnml index 0be11fba13d..05ce1e42482 100755 --- a/src/components/data_comps/docn/cime_config/buildnml +++ b/src/components/data_comps/docn/cime_config/buildnml @@ -166,7 +166,7 @@ def buildnml(case, caseroot, compname): # Loop over instances #---------------------------------------------------- for inst_counter in range(1, ninst+1): - + nmlgen.new_instance() # determine instance string inst_string = "" if ninst > 1: diff --git a/src/components/data_comps/drof/cime_config/buildnml b/src/components/data_comps/drof/cime_config/buildnml index ae3c683496f..5751b2d5aa9 100755 --- a/src/components/data_comps/drof/cime_config/buildnml +++ b/src/components/data_comps/drof/cime_config/buildnml @@ -164,7 +164,7 @@ def buildnml(case, caseroot, compname): # Loop over instances #---------------------------------------------------- for inst_counter in range(1, ninst+1): - + nmlgen.new_instance() # determine instance string inst_string = "" if ninst > 1: diff --git a/src/components/data_comps/dwav/cime_config/buildnml b/src/components/data_comps/dwav/cime_config/buildnml index cb9ec308ea5..aaf2f293508 100755 --- a/src/components/data_comps/dwav/cime_config/buildnml +++ b/src/components/data_comps/dwav/cime_config/buildnml @@ -170,7 +170,7 @@ def buildnml(case, caseroot, compname): # Loop over instances #---------------------------------------------------- for inst_counter in range(1, ninst+1): - + nmlgen.new_instance() # determine instance string inst_string = "" if ninst > 1: From 906c66a28651151b214aab5cc077bd5d2e05ce82 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 17 Mar 2017 15:12:24 -0600 Subject: [PATCH 07/25] remove commented code --- scripts/lib/CIME/nmlgen.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/scripts/lib/CIME/nmlgen.py b/scripts/lib/CIME/nmlgen.py index 14aaaa74f02..e59d7d118e2 100644 --- a/scripts/lib/CIME/nmlgen.py +++ b/scripts/lib/CIME/nmlgen.py @@ -530,14 +530,8 @@ def add_default(self, name, value=None, ignore_abs_path=None): # Use this to see if we need to raise an error when nothing is found. have_value = False - # Check for existing value. current_literals = self._namelist.get_variable_value(group, name) -# I may have broken something here??? -# if current_literals != [""]: -# have_value = True - # Do not proceed further since this has been obtained the -infile contents -# return # Check for input argument. if value is not None: From c106652d91ed5c08c627d2e7489316b7bd229178 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 20 Mar 2017 11:35:44 -0600 Subject: [PATCH 08/25] add space before return --- scripts/lib/CIME/namelist.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/lib/CIME/namelist.py b/scripts/lib/CIME/namelist.py index 381927b6218..797c2a68ac4 100644 --- a/scripts/lib/CIME/namelist.py +++ b/scripts/lib/CIME/namelist.py @@ -1465,6 +1465,7 @@ def _parse_variable_name(self, allow_equals=True): self._nameindex[vname].append(nindex) else: self._nameindex[vname] = [nindex] + return text.lower() def _parse_character_literal(self): From 31c5f9523d85b26a6292c4aa49f863ed01da0533 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 20 Mar 2017 13:53:11 -0600 Subject: [PATCH 09/25] added error for values out of range --- scripts/lib/CIME/namelist.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/lib/CIME/namelist.py b/scripts/lib/CIME/namelist.py index 797c2a68ac4..5ea95c14f7c 100644 --- a/scripts/lib/CIME/namelist.py +++ b/scripts/lib/CIME/namelist.py @@ -112,7 +112,7 @@ # Fortran syntax regular expressions. # Variable names. -FORTRAN_NAME_REGEX = re.compile(r"^[a-z][a-z0-9_]{0,62}(\((\d+)\))?$", re.IGNORECASE) +FORTRAN_NAME_REGEX = re.compile(r"^[a-z][a-z0-9_]{0,62}(\(([+-]?\d+)\))?$", re.IGNORECASE) FORTRAN_LITERAL_REGEXES = {} # Integer literals. _int_re_string = r"(\+|-)?[0-9]+" @@ -1459,6 +1459,10 @@ def _parse_variable_name(self, allow_equals=True): nindex = fortran_name_index(text_check) if nindex is not None: + if nindex <= 0 or nindex > self._len: + raise _NamelistParseError("index value of %s is not supported in variable %s"% + (nindex, str(text))) + text = text_check[:text.find('(')] vname = text.lower() if vname in self._nameindex: From 3cfe6cca5dc56cd18c803bc30588b93cba8bcc1d Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 21 Mar 2017 18:18:12 -0600 Subject: [PATCH 10/25] finish abandoned branch --- scripts/lib/CIME/XML/namelist_definition.py | 6 +- scripts/lib/CIME/namelist.py | 146 ++++++++++---------- 2 files changed, 77 insertions(+), 75 deletions(-) diff --git a/scripts/lib/CIME/XML/namelist_definition.py b/scripts/lib/CIME/XML/namelist_definition.py index b9f76f2aaa8..67b4168e1af 100644 --- a/scripts/lib/CIME/XML/namelist_definition.py +++ b/scripts/lib/CIME/XML/namelist_definition.py @@ -14,7 +14,7 @@ from CIME.namelist import fortran_namelist_base_value, \ is_valid_fortran_namelist_literal, character_literal_to_string, \ - expand_literal_list, Namelist + expand_literal_list, Namelist, get_variable_name from CIME.XML.standard_module_setup import * from CIME.XML.entry_id import EntryID @@ -344,7 +344,7 @@ def validate(self, namelist,filename=None): # Check that the variable is defined... self._expect_variable_in_definition(variable_name, variable_template) - # Check if can actually change this variable via filename change + # Check if can actuax1lly change this variable via filename change if filename is not None: self._user_modifiable_in_variable_definition(variable_name) @@ -379,7 +379,7 @@ def dict_to_namelist(self, dict_, filename=None): groups = {} for variable_name in dict_: variable_lc = variable_name.lower() - self._expect_variable_in_definition(variable_lc, variable_template) + self._expect_variable_in_definition(get_variable_name(variable_lc), variable_template) group_name = self.get_group(variable_lc) expect (group_name is not None, "No group found for var %s"%variable_lc) if group_name not in groups: diff --git a/scripts/lib/CIME/namelist.py b/scripts/lib/CIME/namelist.py index 5ea95c14f7c..e55a0c2aeac 100644 --- a/scripts/lib/CIME/namelist.py +++ b/scripts/lib/CIME/namelist.py @@ -112,7 +112,7 @@ # Fortran syntax regular expressions. # Variable names. -FORTRAN_NAME_REGEX = re.compile(r"^[a-z][a-z0-9_]{0,62}(\(([+-]?\d+)\))?$", re.IGNORECASE) +FORTRAN_NAME_REGEX = re.compile(r"(^[a-z][a-z0-9_]{0,62})(\([+-]?\d*:?[+-]?\d*:?[+-]?\d*\))?$", re.IGNORECASE) FORTRAN_LITERAL_REGEXES = {} # Integer literals. _int_re_string = r"(\+|-)?[0-9]+" @@ -156,6 +156,18 @@ def is_valid_fortran_name(string): True >>> is_valid_fortran_name("A") True + >>> is_valid_fortran_name("A(4)") + True + >>> is_valid_fortran_name("A(::)") + True + >>> is_valid_fortran_name("A(1:2:3)") + True + >>> is_valid_fortran_name("A(1::)") + True + >>> is_valid_fortran_name("A(:-2:)") + True + >>> is_valid_fortran_name("A(1::+3)") + True >>> is_valid_fortran_name("2") False >>> is_valid_fortran_name("_") @@ -171,20 +183,10 @@ def is_valid_fortran_name(string): """ return FORTRAN_NAME_REGEX.search(string) is not None -def fortran_name_index(string): - """ Get name index if provided - - >>> fortran_name_index("foo(2)") - 2 - >>> fortran_name_index("bar") - - >>> fortran_name_index("foo(a)") - - """ - m = FORTRAN_NAME_REGEX.search(string) - if m is not None and m.group(2) is not None: - return int(m.group(2)) - +def get_variable_name(full_var): + """ Return the variable name with all array syntax info removed """ + m = FORTRAN_NAME_REGEX.search(full_var) + return m.group(1) def fortran_namelist_base_value(string): r"""Strip off whitespace and repetition syntax from a namelist value. @@ -615,7 +617,7 @@ def literal_to_python_value(literal, type_=None): return float(literal) -def expand_literal_list(literals, nindex=None): +def expand_literal_list(literals): """Expands a list of literal values to get rid of repetition syntax. >>> expand_literal_list([]) @@ -624,8 +626,6 @@ def expand_literal_list(literals, nindex=None): ['true'] >>> expand_literal_list(['1', '2', 'f*', '3*3', '5']) ['1', '2', 'f*', '3', '3', '3', '5'] - >>> expand_literal_list(['1'],nindex=2) - ['', '1'] >>> expand_literal_list([u'2*f*']) [u'f*', u'f*'] """ @@ -634,11 +634,9 @@ def expand_literal_list(literals, nindex=None): if FORTRAN_REPEAT_PREFIX_REGEX.search(literal) is not None: num, _, value = literal.partition('*') expanded += int(num) * [value] - elif nindex is not None: - expanded = ['']*(nindex-1) - expanded.append(literal) else: expanded.append(literal) + return expanded @@ -692,7 +690,7 @@ def compress_literal_list(literals): compressed.append(str(literal)) return compressed -def merge_literal_lists(default, overwrite, dindex=None, oindex=None): +def merge_literal_lists(default, overwrite): """Merge two lists of literal value strings. The `overwrite` values have higher precedence, so will overwrite the @@ -714,22 +712,12 @@ def merge_literal_lists(default, overwrite, dindex=None, oindex=None): ['true'] >>> merge_literal_lists(['true'], []) ['true'] - >>> merge_literal_lists(['true'], ['false'], dindex=2) - ['false', 'true'] - >>> merge_literal_lists(['true'], [], dindex=2) - ['', 'true'] - >>> merge_literal_lists([], ['foo'], oindex=2) - ['', 'foo'] - >>> merge_literal_lists(['true','true','true'], ['false'], oindex=2) - ['true', 'false', 'true'] - >>> merge_literal_lists(['false'],['true','true','true'], dindex=2) - ['true', 'true', 'true'] >>> merge_literal_lists(['3*false', '3*true'], ['true', '4*', 'false']) ['true', 'false', 'false', 'true', 'true', 'false'] """ merged = [] - default = expand_literal_list(default, dindex) - overwrite = expand_literal_list(overwrite, oindex) + default = expand_literal_list(default) + overwrite = expand_literal_list(overwrite) for default_elem, elem in zip(default, overwrite): if elem == '': merged.append(default_elem) @@ -852,6 +840,35 @@ def get_group_names(self): """ return self._groups.keys() + def get_variable_qualified_names(self, group_name): + """Return a list of all variables in the given namelist group. + + If the specified group is not in the namelist, returns an empty list. + + >>> Namelist().get_variable_names('foo') + [] + >>> x = parse(text='&foo bar=,bazz=true,bazz(2)=fred,bang=6*""/') + >>> sorted(x.get_variable_names('fOo')) + [u'bang', u'bar', u'bazz'] + >>> x = parse(text='&foo bar=,bazz=true,bang=6*""/') + >>> sorted(x.get_variable_names('fOo')) + [u'bang', u'bar', u'bazz'] + >>> x = parse(text='&foo bar(::)=,bazz=false,bazz(2)=true,bazz(:2:)=6*""/') + >>> sorted(x.get_variable_names('fOo')) + [u'bar', u'bazz'] + """ + group_name = group_name.lower() + if group_name not in self._groups: + return [] + var_full_names = self._groups[group_name].keys() + varnames = [] + for var in var_full_names: + varname = get_variable_name(var) + if varname not in varnames: + varnames.append(varname) + + return varnames + def get_variable_names(self, group_name): """Return a list of all variables in the given namelist group. @@ -859,9 +876,15 @@ def get_variable_names(self, group_name): >>> Namelist().get_variable_names('foo') [] + >>> x = parse(text='&foo bar=,bazz=true,bazz(2)=fred,bang=6*""/') + >>> sorted(x.get_variable_names('fOo')) + [u'bang', u'bar', u'bazz'] >>> x = parse(text='&foo bar=,bazz=true,bang=6*""/') >>> sorted(x.get_variable_names('fOo')) [u'bang', u'bar', u'bazz'] + >>> x = parse(text='&foo bar(::)=,bazz=false,bazz(2)=true,bazz(:2:)=6*""/') + >>> sorted(x.get_variable_names('fOo')) + [u'bar', u'bazz'] """ group_name = group_name.lower() if group_name not in self._groups: @@ -887,6 +910,7 @@ def get_variable_value(self, group_name, variable_name): if group_name not in self._groups or \ variable_name not in self._groups[group_name]: return [u''] + return self._groups[group_name][variable_name] def get_value(self, variable_name): @@ -902,8 +926,8 @@ def get_value(self, variable_name): SystemExit: ERROR: Namelist.get_value: Variable %s is present in multiple groups: [u'bazz', u'foo'] >>> parse(text='&foo bar=1 / &bazz /').get_value('Bar') [u'1'] - >>> parse(text='&foo bar(2)=1 / &bazz /').get_value('Bar') - ['', u'1'] + >>> parse(text='&foo bar(2)=1 / &bazz /').get_value('Bar(2)') + [u'1'] >>> parse(text='&foo / &bazz /').get_value('bar') [u''] """ @@ -934,6 +958,7 @@ def set_variable_value(self, group_name, variable_name, value): """ group_name = group_name.lower() variable_name = variable_name.lower() + if group_name not in self._groups: self._groups[group_name] = {} self._groups[group_name][variable_name] = value @@ -1418,13 +1443,15 @@ def _parse_variable_name(self, allow_equals=True): ... _NamelistEOF: Unexpected end of file encountered in namelist. (At line 1, column 3) >>> _NamelistParser('foo(2)= ')._parse_variable_name() - u'foo' + u'foo(2)' >>> _NamelistParser('abc ')._parse_variable_name() u'abc' >>> _NamelistParser('ABC ')._parse_variable_name() u'abc' >>> _NamelistParser('abc\n')._parse_variable_name() u'abc' + >>> _NamelistParser('abc(1:2:3)\n')._parse_variable_name() + u'abc(1:2:3)' >>> _NamelistParser('abc=')._parse_variable_name() u'abc' >>> _NamelistParser('abc, ')._parse_variable_name() @@ -1454,22 +1481,6 @@ def _parse_variable_name(self, allow_equals=True): raise _NamelistParseError("%r is not a valid variable name at %s" % (str(text), self._line_col_string())) - # a particular value of a namelist array may be assigned using syntax - # foo(2) = 7 - - nindex = fortran_name_index(text_check) - if nindex is not None: - if nindex <= 0 or nindex > self._len: - raise _NamelistParseError("index value of %s is not supported in variable %s"% - (nindex, str(text))) - - text = text_check[:text.find('(')] - vname = text.lower() - if vname in self._nameindex: - self._nameindex[vname].append(nindex) - else: - self._nameindex[vname] = [nindex] - return text.lower() def _parse_character_literal(self): @@ -1823,7 +1834,7 @@ def _parse_name_and_values(self, allow_eof_end=False): >>> _NamelistParser("foo='bar' /")._parse_name_and_values() (u'foo', [u"'bar'"]) >>> _NamelistParser("foo(3)='bar' /")._parse_name_and_values() - (u'foo', [u"'bar'"]) + (u'foo(3)', [u"'bar'"]) >>> _NamelistParser("foo ='bar' /")._parse_name_and_values() (u'foo', [u"'bar'"]) >>> _NamelistParser("foo=\n'bar' /")._parse_name_and_values() @@ -1847,7 +1858,7 @@ def _parse_name_and_values(self, allow_eof_end=False): >>> _NamelistParser("foo=,,'bazz',6* ")._parse_name_and_values(allow_eof_end=True) (u'foo', [u'', u'', u"'bazz'", u'6*']) >>> _NamelistParser("foo(3)='bazz'")._parse_name_and_values(allow_eof_end=True) - (u'foo', [u"'bazz'"]) + (u'foo(3)', [u"'bazz'"]) >>> _NamelistParser("foo=")._parse_name_and_values() Traceback (most recent call last): ... @@ -1940,20 +1951,17 @@ def _parse_namelist_group(self): self._eat_whitespace() while self._curr() != '/': name, values = self._parse_name_and_values() - oindex = None dsettings = [] - if name in self._nameindex: - oindex = self._nameindex[name].pop(0) if self._groupless: if name in self._settings: dsettings = self._settings[name] - values = merge_literal_lists(dsettings, values, oindex=oindex) + values = merge_literal_lists(dsettings, values) self._settings[name] = values else: group = self._settings[group_name] if name in group: dsettings = group[name] - values = merge_literal_lists(dsettings, values, oindex=oindex) + values = merge_literal_lists(dsettings, values) group[name] = values def parse_namelist(self): @@ -1975,9 +1983,9 @@ def parse_namelist(self): >>> _NamelistParser("&group1\n foo='bar','bazz'\n,, foo2=2*5\n / &group2 /").parse_namelist() {u'group1': {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'5', u'5']}, u'group2': {}} >>> _NamelistParser("!blah \n foo='bar','bazz'\n,, foo2=2*5\n ", groupless=True).parse_namelist() - {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'5', u'5']} + {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'2*5']} >>> _NamelistParser("!blah \n foo='bar','bazz'\n,, foo2=2*5,6\n ", groupless=True).parse_namelist() - {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'5', u'5', u'6']} + {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'2*5', u'6']} >>> _NamelistParser("!blah \n foo='bar'", groupless=True).parse_namelist() {u'foo': [u"'bar'"]} >>> _NamelistParser("foo='bar', foo='bazz'", groupless=True).parse_namelist() @@ -1985,11 +1993,11 @@ def parse_namelist(self): >>> _NamelistParser("foo='bar', foo=", groupless=True).parse_namelist() {u'foo': [u"'bar'"]} >>> _NamelistParser("foo='bar', foo(3)='bazz'", groupless=True).parse_namelist() - {u'foo': [u"'bar'", '', u"'bazz'"]} + {u'foo': [u"'bar'"], u'foo(3)': [u"'bazz'"]} >>> _NamelistParser("foo(2)='bar'", groupless=True).parse_namelist() - {u'foo': ['', u"'bar'"]} + {u'foo(2)': [u"'bar'"]} >>> _NamelistParser("foo(2)='bar', foo(3)='bazz'", groupless=True).parse_namelist() - {u'foo': ['', u"'bar'", u"'bazz'"]} + {u'foo(2)': [u"'bar'"], u'foo(3)': [u"'bazz'"]} """ # Return empty dictionary for empty files. if self._len == 0: @@ -2004,14 +2012,8 @@ def parse_namelist(self): if self._groupless and self._curr() != '&': while self._pos < self._len: name, values = self._parse_name_and_values(allow_eof_end=True) - oindex = None - if name in self._nameindex: - oindex = self._nameindex[name].pop(0) - dsettings = [] if name in self._settings: - dsettings = self._settings[name] - values = merge_literal_lists(dsettings, values, - oindex=oindex) + values = merge_literal_lists(self._settings[name], values) self._settings[name] = values return self._settings # Loop over namelist groups in the file. From 5a12888604ee0ff4ab39533e610313a584792bb1 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 22 Mar 2017 15:19:34 -0600 Subject: [PATCH 11/25] seems to be working now --- scripts/lib/CIME/XML/namelist_definition.py | 16 +- scripts/lib/CIME/namelist.py | 176 +++++++++++++++----- scripts/lib/CIME/nmlgen.py | 8 +- 3 files changed, 148 insertions(+), 52 deletions(-) diff --git a/scripts/lib/CIME/XML/namelist_definition.py b/scripts/lib/CIME/XML/namelist_definition.py index 67b4168e1af..305759c78d6 100644 --- a/scripts/lib/CIME/XML/namelist_definition.py +++ b/scripts/lib/CIME/XML/namelist_definition.py @@ -14,7 +14,7 @@ from CIME.namelist import fortran_namelist_base_value, \ is_valid_fortran_namelist_literal, character_literal_to_string, \ - expand_literal_list, Namelist, get_variable_name + expand_literal_list, Namelist, get_fortran_name_only from CIME.XML.standard_module_setup import * from CIME.XML.entry_id import EntryID @@ -342,21 +342,22 @@ def validate(self, namelist,filename=None): for group_name in namelist.get_group_names(): for variable_name in namelist.get_variable_names(group_name): # Check that the variable is defined... - self._expect_variable_in_definition(variable_name, variable_template) + qualified_variable_name = get_fortran_name_only(variable_name) + self._expect_variable_in_definition(qualified_variable_name, variable_template) # Check if can actuax1lly change this variable via filename change if filename is not None: - self._user_modifiable_in_variable_definition(variable_name) + self._user_modifiable_in_variable_definition(qualified_variable_name) # and has the right group name... - var_group = self.get_group(variable_name) + var_group = self.get_group(qualified_variable_name) expect(var_group == group_name, (variable_template + " is in a group named %r, but should be in %r.") % (str(variable_name), str(group_name), str(var_group))) # and has a valid value. value = namelist.get_variable_value(group_name, variable_name) - expect(self.is_valid_value(variable_name, value), + expect(self.is_valid_value(qualified_variable_name, value), (variable_template + " has invalid value %r.") % (str(variable_name), [str(scalar) for scalar in value])) @@ -379,8 +380,9 @@ def dict_to_namelist(self, dict_, filename=None): groups = {} for variable_name in dict_: variable_lc = variable_name.lower() - self._expect_variable_in_definition(get_variable_name(variable_lc), variable_template) - group_name = self.get_group(variable_lc) + qualified_varname = get_fortran_name_only(variable_lc) + self._expect_variable_in_definition(qualified_varname, variable_template) + group_name = self.get_group(qualified_varname) expect (group_name is not None, "No group found for var %s"%variable_lc) if group_name not in groups: groups[group_name] = {} diff --git a/scripts/lib/CIME/namelist.py b/scripts/lib/CIME/namelist.py index e55a0c2aeac..05dc1c1433b 100644 --- a/scripts/lib/CIME/namelist.py +++ b/scripts/lib/CIME/namelist.py @@ -112,7 +112,21 @@ # Fortran syntax regular expressions. # Variable names. -FORTRAN_NAME_REGEX = re.compile(r"(^[a-z][a-z0-9_]{0,62})(\([+-]?\d*:?[+-]?\d*:?[+-]?\d*\))?$", re.IGNORECASE) +#FORTRAN_NAME_REGEX = re.compile(r"(^[a-z][a-z0-9_]{0,62})(\([+-]?\d*:?[+-]?\d*:?[+-]?\d*\))?$", re.IGNORECASE) +FORTRAN_NAME_REGEX = re.compile(r"""(^[a-z][a-z0-9_]{0,62}) # The variable name + (\( # begin optional index expression + (([+-]?\d+) # Single valued index + | # or + (([+-]?\d+)?:([+-]?\d+)?:?([+-]?\d+)?)) # colon seperated triplet + \))?\s*$""" # end optional index expression + , re.IGNORECASE | re.VERBOSE) + + + + + + + FORTRAN_LITERAL_REGEXES = {} # Integer literals. _int_re_string = r"(\+|-)?[0-9]+" @@ -183,11 +197,54 @@ def is_valid_fortran_name(string): """ return FORTRAN_NAME_REGEX.search(string) is not None -def get_variable_name(full_var): - """ Return the variable name with all array syntax info removed """ +def get_fortran_name_only(full_var): + """ remove array section if any and return only the variable name + >>> get_fortran_name_only('foo') + 'foo' + >>> get_fortran_name_only('foo(3)') + 'foo' + >>> get_fortran_name_only('foo(::)') + 'foo' + >>> get_fortran_name_only('foo(1::)') + 'foo' + >>> get_fortran_name_only('foo(:+2:)') + 'foo' + >>> get_fortran_name_only('foo(::-3)') + 'foo' + >>> get_fortran_name_only('foo(::)') + 'foo' + """ m = FORTRAN_NAME_REGEX.search(full_var) return m.group(1) +def get_fortran_variable_indices(varname, varlen=1): + """ get indices from a fortran namelist variable as a triplet of minindex, maxindex and step + + >>> get_fortran_variable_indices('foo(3)') + (3, 3, 1) + >>> get_fortran_variable_indices('foo(1:2:3)') + (1, 2, 3) + >>> get_fortran_variable_indices('foo(::)', varlen=4) + (1, 4, 1) + """ + m = FORTRAN_NAME_REGEX.search(varname) + (minindex, maxindex, step) = (1, varlen, 1) + + if m.group(4) is not None: + minindex = int(m.group(4)) + maxindex = minindex + step = 1 + + elif m.group(5) is not None: + if m.group(6) is not None: + minindex = int(m.group(6)) + if m.group(7) is not None: + maxindex = int(m.group(7)) + if m.group(8) is not None: + step = int(m.group(8)) + + return (minindex, maxindex, step) + def fortran_namelist_base_value(string): r"""Strip off whitespace and repetition syntax from a namelist value. @@ -718,6 +775,7 @@ def merge_literal_lists(default, overwrite): merged = [] default = expand_literal_list(default) overwrite = expand_literal_list(overwrite) + for default_elem, elem in zip(default, overwrite): if elem == '': merged.append(default_elem) @@ -840,34 +898,34 @@ def get_group_names(self): """ return self._groups.keys() - def get_variable_qualified_names(self, group_name): - """Return a list of all variables in the given namelist group. - - If the specified group is not in the namelist, returns an empty list. - - >>> Namelist().get_variable_names('foo') - [] - >>> x = parse(text='&foo bar=,bazz=true,bazz(2)=fred,bang=6*""/') - >>> sorted(x.get_variable_names('fOo')) - [u'bang', u'bar', u'bazz'] - >>> x = parse(text='&foo bar=,bazz=true,bang=6*""/') - >>> sorted(x.get_variable_names('fOo')) - [u'bang', u'bar', u'bazz'] - >>> x = parse(text='&foo bar(::)=,bazz=false,bazz(2)=true,bazz(:2:)=6*""/') - >>> sorted(x.get_variable_names('fOo')) - [u'bar', u'bazz'] - """ - group_name = group_name.lower() - if group_name not in self._groups: - return [] - var_full_names = self._groups[group_name].keys() - varnames = [] - for var in var_full_names: - varname = get_variable_name(var) - if varname not in varnames: - varnames.append(varname) - - return varnames + # def get_variable_qualified_names(self, group_name): + # """Return a list of all variables in the given namelist group. + + # If the specified group is not in the namelist, returns an empty list. + + # >>> Namelist().get_variable_names('foo') + # [] + # >>> x = parse(text='&foo bar=,bazz=true,bazz(2)=fred,bang=6*""/') + # >>> sorted(x.get_variable_names('fOo')) + # [u'bang', u'bar', u'bazz'] + # >>> x = parse(text='&foo bar=,bazz=true,bang=6*""/') + # >>> sorted(x.get_variable_names('fOo')) + # [u'bang', u'bar', u'bazz'] + # >>> x = parse(text='&foo bar(::)=,bazz=false,bazz(2)=true,bazz(:2:)=6*""/') + # >>> sorted(x.get_variable_names('fOo')) + # [u'bar', u'bazz'] + # """ + # group_name = group_name.lower() + # if group_name not in self._groups: + # return [] + # var_full_names = self._groups[group_name].keys() + # varnames = [] + # for var in var_full_names: + # varname = get_variable_name(var) + # if varname not in varnames: + # varnames.append(varname) + + # return varnames def get_variable_names(self, group_name): """Return a list of all variables in the given namelist group. @@ -878,13 +936,13 @@ def get_variable_names(self, group_name): [] >>> x = parse(text='&foo bar=,bazz=true,bazz(2)=fred,bang=6*""/') >>> sorted(x.get_variable_names('fOo')) - [u'bang', u'bar', u'bazz'] + [u'bang', u'bar', u'bazz', u'bazz(2)'] >>> x = parse(text='&foo bar=,bazz=true,bang=6*""/') >>> sorted(x.get_variable_names('fOo')) [u'bang', u'bar', u'bazz'] >>> x = parse(text='&foo bar(::)=,bazz=false,bazz(2)=true,bazz(:2:)=6*""/') >>> sorted(x.get_variable_names('fOo')) - [u'bar', u'bazz'] + [u'bar(::)', u'bazz', u'bazz(2)', u'bazz(:2:)'] """ group_name = group_name.lower() if group_name not in self._groups: @@ -942,26 +1000,53 @@ def get_value(self, variable_name): else: return [u''] - def set_variable_value(self, group_name, variable_name, value): + def set_variable_value(self, group_name, variable_name, value, var_size=1): """Set the value of the specified variable. >>> x = parse(text='&foo bar=1 /') + >>> x.get_variable_value('foo', 'bar') + [u'1'] + >>> x.set_variable_value('foo', 'bar(2)', [u'3'], var_size=4) + >>> x.get_variable_value('foo', 'bar') + [u'1', u'3'] >>> x.set_variable_value('foo', 'bar', [u'2']) + >>> x.get_variable_value('foo', 'bar') + [u'2', u'3'] >>> x.set_variable_value('foo', 'bazz', [u'3']) >>> x.set_variable_value('Brack', 'baR', [u'4']) - >>> x.get_variable_value('foo', 'bar') - [u'2'] >>> x.get_variable_value('foo', 'bazz') [u'3'] >>> x.get_variable_value('brack', 'bar') [u'4'] """ group_name = group_name.lower() - variable_name = variable_name.lower() + +# print "BEFORE name %s value %s"%(variable_name, value) + minindex, maxindex, step = get_fortran_variable_indices(variable_name, var_size) + variable_name = get_fortran_name_only(variable_name.lower()) + + expect(minindex > 0, "CIME indices < 1 not supported in CIME interface to fortran namelists minindex=%s"%minindex) if group_name not in self._groups: self._groups[group_name] = {} - self._groups[group_name][variable_name] = value + tlen = 1 + if variable_name in self._groups[group_name]: + tlen = len(self._groups[group_name][variable_name]) + else: + tlen = 0 + self._groups[group_name][variable_name] = [] + + if minindex > tlen: + self._groups[group_name][variable_name].extend(['']*(minindex-tlen-1)) + + for i in range(minindex-1, maxindex, step): + if i < tlen: + self._groups[group_name][variable_name][i] = value.pop(0) + else: + self._groups[group_name][variable_name].append(value.pop(0)) + if len(value) == 0: + break + def delete_variable(self, group_name, variable_name): """Delete a variable from a specified group. @@ -992,6 +1077,8 @@ def merge_nl(self, other, overwrite=False): >>> x = parse(text='&foo bar=1 bazz=,2 brat=3/') >>> y = parse(text='&foo bar=2 bazz=3*1 baker=4 / &foo2 barter=5 /') + >>> y.get_value('bazz') + [u'1', u'1', u'1'] >>> x.merge_nl(y) >>> sorted(x.get_group_names()) [u'foo', u'foo2'] @@ -1009,7 +1096,6 @@ def merge_nl(self, other, overwrite=False): [u'4'] >>> x.get_value('barter') [u'5'] - >>> x = parse(text='&foo bar=1 bazz=,2 brat=3/') >>> y = parse(text='&foo bar=2 bazz=3*1 baker=4 / &foo2 barter=5 /') >>> x.merge_nl(y, overwrite=True) @@ -1040,7 +1126,8 @@ def merge_nl(self, other, overwrite=False): merged_val = merge_literal_lists(self_val, other_val) else: merged_val = merge_literal_lists(other_val, self_val) - self.set_variable_value(group_name, variable_name, merged_val) + self.set_variable_value(group_name, variable_name, merged_val, + var_size=len(merged_val)) def write(self, out_file, groups=None, append=False, format_='nml', sorted_groups=True): """Write a Fortran namelist to a file. @@ -1183,7 +1270,6 @@ def __init__(self, text, groupless=False): # Fortran allows setting a particular index of an array # such as foo(2)='k' # this dict is set to that value if used. - self._nameindex = {} self._groupless = groupless def _line_col_string(self): @@ -1450,6 +1536,10 @@ def _parse_variable_name(self, allow_equals=True): u'abc' >>> _NamelistParser('abc\n')._parse_variable_name() u'abc' + >>> _NamelistParser('abc%fred\n')._parse_variable_name() + u'abc%fred' + >>> _NamelistParser('abc(2)@fred\n')._parse_variable_name() + u'abc(2)@fred' >>> _NamelistParser('abc(1:2:3)\n')._parse_variable_name() u'abc(1:2:3)' >>> _NamelistParser('abc=')._parse_variable_name() @@ -1477,11 +1567,13 @@ def _parse_variable_name(self, allow_equals=True): text_check = re.sub('@.+$', "", text) else: text_check = text + if not is_valid_fortran_name(text_check): raise _NamelistParseError("%r is not a valid variable name at %s" % (str(text), self._line_col_string())) + name = text.lower() - return text.lower() + return name def _parse_character_literal(self): """Parse and return a character literal (a string). diff --git a/scripts/lib/CIME/nmlgen.py b/scripts/lib/CIME/nmlgen.py index e59d7d118e2..c6b37187aca 100644 --- a/scripts/lib/CIME/nmlgen.py +++ b/scripts/lib/CIME/nmlgen.py @@ -211,7 +211,8 @@ def set_value(self, name, value): """ var_group = self._definition.get_group(name) literals = self._to_namelist_literals(name, value) - self._namelist.set_variable_value(var_group, name, literals) + _, _, var_size, = self._definition.split_type_string(name) + self._namelist.set_variable_value(var_group, name, literals, var_size) def get_default(self, name, config=None, allow_none=False): """Get the value of a variable from the namelist definition file. @@ -550,7 +551,8 @@ def add_default(self, name, value=None, ignore_abs_path=None): # Go through file names and prepend input data root directory for # absolute pathnames. - if ignore_abs_path is None: + var_type, _, var_size = self._definition.split_type_string(name) + if var_type == "char" and ignore_abs_path is None: var_input_pathname = self._definition.get_input_pathname(name) if var_input_pathname == 'abs': current_literals = expand_literal_list(current_literals) @@ -570,7 +572,7 @@ def add_default(self, name, value=None, ignore_abs_path=None): current_literals = compress_literal_list(current_literals) # Set the new value. - self._namelist.set_variable_value(group, name, current_literals) + self._namelist.set_variable_value(group, name, current_literals, var_size) def create_shr_strdata_nml(self): """Set defaults for `shr_strdata_nml` variables other than the variable domainfile """ From d08537077fd45f4ba78daeac415a6f62d764bc7b Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 23 Mar 2017 06:57:50 -0600 Subject: [PATCH 12/25] fix step > 1 --- scripts/lib/CIME/namelist.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/lib/CIME/namelist.py b/scripts/lib/CIME/namelist.py index 05dc1c1433b..5be82f7d15e 100644 --- a/scripts/lib/CIME/namelist.py +++ b/scripts/lib/CIME/namelist.py @@ -1018,6 +1018,9 @@ def set_variable_value(self, group_name, variable_name, value, var_size=1): [u'3'] >>> x.get_variable_value('brack', 'bar') [u'4'] + >>> x.set_variable_value('foo', 'red(2:6:2)', [u'2', u'4', u'6'], var_size=12) + >>> x.get_variable_value('foo', 'red') + ['', u'2', '', u'4', '', u'6'] """ group_name = group_name.lower() @@ -1033,8 +1036,8 @@ def set_variable_value(self, group_name, variable_name, value, var_size=1): if variable_name in self._groups[group_name]: tlen = len(self._groups[group_name][variable_name]) else: - tlen = 0 - self._groups[group_name][variable_name] = [] + tlen = 1 + self._groups[group_name][variable_name] = [''] if minindex > tlen: self._groups[group_name][variable_name].extend(['']*(minindex-tlen-1)) @@ -1043,7 +1046,10 @@ def set_variable_value(self, group_name, variable_name, value, var_size=1): if i < tlen: self._groups[group_name][variable_name][i] = value.pop(0) else: + if tlen < i: + self._groups[group_name][variable_name].extend(['']*(i-tlen)) self._groups[group_name][variable_name].append(value.pop(0)) + tlen = len(self._groups[group_name][variable_name]) if len(value) == 0: break From a46759ab42e29b50b121c076260cc079b8106ac9 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 23 Mar 2017 07:04:38 -0600 Subject: [PATCH 13/25] character not char --- scripts/lib/CIME/nmlgen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/lib/CIME/nmlgen.py b/scripts/lib/CIME/nmlgen.py index c6b37187aca..392137e90db 100644 --- a/scripts/lib/CIME/nmlgen.py +++ b/scripts/lib/CIME/nmlgen.py @@ -552,7 +552,7 @@ def add_default(self, name, value=None, ignore_abs_path=None): # Go through file names and prepend input data root directory for # absolute pathnames. var_type, _, var_size = self._definition.split_type_string(name) - if var_type == "char" and ignore_abs_path is None: + if var_type == "character" and ignore_abs_path is None: var_input_pathname = self._definition.get_input_pathname(name) if var_input_pathname == 'abs': current_literals = expand_literal_list(current_literals) From 757bc28f359ec511d89f4af456993838e36ff26f Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 23 Mar 2017 14:02:22 -0600 Subject: [PATCH 14/25] need a special case for cice --- scripts/lib/CIME/nmlgen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/lib/CIME/nmlgen.py b/scripts/lib/CIME/nmlgen.py index 392137e90db..4d148e774cb 100644 --- a/scripts/lib/CIME/nmlgen.py +++ b/scripts/lib/CIME/nmlgen.py @@ -561,7 +561,7 @@ def add_default(self, name, value=None, ignore_abs_path=None): continue file_path = character_literal_to_string(literal) # NOTE - these are hard-coded here and a better way is to make these extensible - if file_path == 'UNSET' or file_path == 'idmap': + if file_path == 'UNSET' or file_path == 'idmap' or file_path == 'unknown_gridcpl_file': continue if file_path == 'null': continue From ed578f005d857e8b987646c5da7701055047a398 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 23 Mar 2017 15:55:04 -0600 Subject: [PATCH 15/25] move new_instance code from buildnml to init_defaults since it is also needed by external components --- scripts/lib/CIME/nmlgen.py | 5 ++++- src/components/data_comps/datm/cime_config/buildnml | 1 - src/components/data_comps/desp/cime_config/buildnml | 1 - src/components/data_comps/dice/cime_config/buildnml | 1 - src/components/data_comps/dlnd/cime_config/buildnml | 1 - src/components/data_comps/docn/cime_config/buildnml | 1 - src/components/data_comps/drof/cime_config/buildnml | 1 - src/components/data_comps/dwav/cime_config/buildnml | 1 - 8 files changed, 4 insertions(+), 8 deletions(-) diff --git a/scripts/lib/CIME/nmlgen.py b/scripts/lib/CIME/nmlgen.py index 4d148e774cb..06dfd81e26f 100644 --- a/scripts/lib/CIME/nmlgen.py +++ b/scripts/lib/CIME/nmlgen.py @@ -96,6 +96,9 @@ def __exit__(self, *_): def init_defaults(self, infiles, config, skip_groups=None, skip_entry_loop=None ): """Return array of names of all definition nodes """ + # first clean out any settings left over from previous calls + self.new_instance() + self._definition.set_nodes(skip_groups=skip_groups) # Determine the array of entry nodes that will be acted upon @@ -561,7 +564,7 @@ def add_default(self, name, value=None, ignore_abs_path=None): continue file_path = character_literal_to_string(literal) # NOTE - these are hard-coded here and a better way is to make these extensible - if file_path == 'UNSET' or file_path == 'idmap' or file_path == 'unknown_gridcpl_file': + if file_path == 'UNSET' or file_path == 'idmap': continue if file_path == 'null': continue diff --git a/src/components/data_comps/datm/cime_config/buildnml b/src/components/data_comps/datm/cime_config/buildnml index 3e2fa726d37..f0ce5fa18fe 100755 --- a/src/components/data_comps/datm/cime_config/buildnml +++ b/src/components/data_comps/datm/cime_config/buildnml @@ -203,7 +203,6 @@ def buildnml(case, caseroot, compname): # Loop over instances #---------------------------------------------------- for inst_counter in range(1, ninst+1): - nmlgen.new_instance() # determine instance string inst_string = "" if ninst > 1: diff --git a/src/components/data_comps/desp/cime_config/buildnml b/src/components/data_comps/desp/cime_config/buildnml index d097974adda..6c938536441 100755 --- a/src/components/data_comps/desp/cime_config/buildnml +++ b/src/components/data_comps/desp/cime_config/buildnml @@ -122,7 +122,6 @@ def buildnml(case, caseroot, compname): # Loop over instances #---------------------------------------------------- for inst_counter in range(1, ninst+1): - nmlgen.new_instance() # determine instance string inst_string = "" if ninst > 1: diff --git a/src/components/data_comps/dice/cime_config/buildnml b/src/components/data_comps/dice/cime_config/buildnml index 341286f0d4e..eef42ff3925 100755 --- a/src/components/data_comps/dice/cime_config/buildnml +++ b/src/components/data_comps/dice/cime_config/buildnml @@ -164,7 +164,6 @@ def buildnml(case, caseroot, compname): # Loop over instances #---------------------------------------------------- for inst_counter in range(1, ninst+1): - nmlgen.new_instance() # determine instance string inst_string = "" if ninst > 1: diff --git a/src/components/data_comps/dlnd/cime_config/buildnml b/src/components/data_comps/dlnd/cime_config/buildnml index ae83c473826..09d88806eb7 100755 --- a/src/components/data_comps/dlnd/cime_config/buildnml +++ b/src/components/data_comps/dlnd/cime_config/buildnml @@ -163,7 +163,6 @@ def buildnml(case, caseroot, compname): # Loop over instances #---------------------------------------------------- for inst_counter in range(1, ninst+1): - nmlgen.new_instance() # determine instance string inst_string = "" if ninst > 1: diff --git a/src/components/data_comps/docn/cime_config/buildnml b/src/components/data_comps/docn/cime_config/buildnml index 05ce1e42482..2afb78283e5 100755 --- a/src/components/data_comps/docn/cime_config/buildnml +++ b/src/components/data_comps/docn/cime_config/buildnml @@ -166,7 +166,6 @@ def buildnml(case, caseroot, compname): # Loop over instances #---------------------------------------------------- for inst_counter in range(1, ninst+1): - nmlgen.new_instance() # determine instance string inst_string = "" if ninst > 1: diff --git a/src/components/data_comps/drof/cime_config/buildnml b/src/components/data_comps/drof/cime_config/buildnml index 5751b2d5aa9..37b0a3a7a3f 100755 --- a/src/components/data_comps/drof/cime_config/buildnml +++ b/src/components/data_comps/drof/cime_config/buildnml @@ -164,7 +164,6 @@ def buildnml(case, caseroot, compname): # Loop over instances #---------------------------------------------------- for inst_counter in range(1, ninst+1): - nmlgen.new_instance() # determine instance string inst_string = "" if ninst > 1: diff --git a/src/components/data_comps/dwav/cime_config/buildnml b/src/components/data_comps/dwav/cime_config/buildnml index aaf2f293508..7a0d67cee2c 100755 --- a/src/components/data_comps/dwav/cime_config/buildnml +++ b/src/components/data_comps/dwav/cime_config/buildnml @@ -170,7 +170,6 @@ def buildnml(case, caseroot, compname): # Loop over instances #---------------------------------------------------- for inst_counter in range(1, ninst+1): - nmlgen.new_instance() # determine instance string inst_string = "" if ninst > 1: From e255277226dd94bcf4ee6f071bbbd506aa4e10c6 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 24 Mar 2017 07:16:37 -0600 Subject: [PATCH 16/25] remove bad line info, better error for multiple dimensions --- scripts/lib/CIME/namelist.py | 141 ++++++++++++++++------------------- 1 file changed, 66 insertions(+), 75 deletions(-) diff --git a/scripts/lib/CIME/namelist.py b/scripts/lib/CIME/namelist.py index 5be82f7d15e..8af9e291fcc 100644 --- a/scripts/lib/CIME/namelist.py +++ b/scripts/lib/CIME/namelist.py @@ -163,6 +163,7 @@ def is_valid_fortran_name(string): 1. The name must start with a letter. 2. All characters in a name must be alphanumeric (or underscores). 3. The maximum name length is 63 characters. + 4. We only handle a single dimension !!! >>> is_valid_fortran_name("") False @@ -182,6 +183,8 @@ def is_valid_fortran_name(string): True >>> is_valid_fortran_name("A(1::+3)") True + >>> is_valid_fortran_name("A(1,3)") + False >>> is_valid_fortran_name("2") False >>> is_valid_fortran_name("_") @@ -898,35 +901,6 @@ def get_group_names(self): """ return self._groups.keys() - # def get_variable_qualified_names(self, group_name): - # """Return a list of all variables in the given namelist group. - - # If the specified group is not in the namelist, returns an empty list. - - # >>> Namelist().get_variable_names('foo') - # [] - # >>> x = parse(text='&foo bar=,bazz=true,bazz(2)=fred,bang=6*""/') - # >>> sorted(x.get_variable_names('fOo')) - # [u'bang', u'bar', u'bazz'] - # >>> x = parse(text='&foo bar=,bazz=true,bang=6*""/') - # >>> sorted(x.get_variable_names('fOo')) - # [u'bang', u'bar', u'bazz'] - # >>> x = parse(text='&foo bar(::)=,bazz=false,bazz(2)=true,bazz(:2:)=6*""/') - # >>> sorted(x.get_variable_names('fOo')) - # [u'bar', u'bazz'] - # """ - # group_name = group_name.lower() - # if group_name not in self._groups: - # return [] - # var_full_names = self._groups[group_name].keys() - # varnames = [] - # for var in var_full_names: - # varname = get_variable_name(var) - # if varname not in varnames: - # varnames.append(varname) - - # return varnames - def get_variable_names(self, group_name): """Return a list of all variables in the given namelist group. @@ -1028,7 +1002,7 @@ def set_variable_value(self, group_name, variable_name, value, var_size=1): minindex, maxindex, step = get_fortran_variable_indices(variable_name, var_size) variable_name = get_fortran_name_only(variable_name.lower()) - expect(minindex > 0, "CIME indices < 1 not supported in CIME interface to fortran namelists minindex=%s"%minindex) + expect(minindex > 0, "Indices < 1 not supported in CIME interface to fortran namelists... lower bound=%s"%minindex) if group_name not in self._groups: self._groups[group_name] = {} @@ -1298,7 +1272,7 @@ def _next(self): >>> _NamelistParser(' ')._next() Traceback (most recent call last): ... - _NamelistEOF: Unexpected end of file encountered in namelist. (At line 1, column 1) + _NamelistEOF: Unexpected end of file encountered in namelist. """ # If at the end of the file, we should raise _NamelistEOF. The easiest # way to do this is to just advance. @@ -1340,11 +1314,11 @@ def _advance(self, nchars=1, check_eof=False): >>> x._advance(1) Traceback (most recent call last): ... - _NamelistEOF: Unexpected end of file encountered in namelist. (At line 3, column 2) + _NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser('abc\n')._advance(4) Traceback (most recent call last): ... - _NamelistEOF: Unexpected end of file encountered in namelist. (At line 2, column 0) + _NamelistEOF: Unexpected end of file encountered in namelist. >>> x = _NamelistParser('ab') >>> x._advance(check_eof=True) False @@ -1369,7 +1343,7 @@ def _advance(self, nchars=1, check_eof=False): if check_eof: return end_of_file elif end_of_file: - raise _NamelistEOF(message="At "+self._line_col_string()) + raise _NamelistEOF(message=None) def _eat_whitespace(self, allow_initial_comment=False): r"""Advance until the next non-whitespace character. @@ -1390,7 +1364,7 @@ def _eat_whitespace(self, allow_initial_comment=False): >>> x._eat_whitespace() Traceback (most recent call last): ... - _NamelistEOF: Unexpected end of file encountered in namelist. (At line 2, column 3) + _NamelistEOF: Unexpected end of file encountered in namelist. >>> x = _NamelistParser(' \n! blah\n ! blah\n a') >>> x._eat_whitespace() True @@ -1447,12 +1421,12 @@ def _eat_comment(self): >>> x._eat_comment() Traceback (most recent call last): ... - _NamelistEOF: Unexpected end of file encountered in namelist. (At line 3, column 8) + _NamelistEOF: Unexpected end of file encountered in namelist. >>> x = _NamelistParser('! foo\n') >>> x._eat_comment() Traceback (most recent call last): ... - _NamelistEOF: Unexpected end of file encountered in namelist. (At line 2, column 0) + _NamelistEOF: Unexpected end of file encountered in namelist. """ if self._curr() != '!': return False @@ -1479,7 +1453,7 @@ def _expect_char(self, chars): >>> x._expect_char('a') Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: expected 'a' but found 'b' at line 1, column 1 + _NamelistParseError: Error in parsing namelist: expected 'a' but found 'b' >>> x._expect_char('ab') """ if self._curr() not in chars: @@ -1487,9 +1461,8 @@ def _expect_char(self, chars): char_description = repr(str(chars)) else: char_description = "one of the characters in %r" % str(chars) - raise _NamelistParseError("expected %s but found %r at %s" % - (char_description, str(self._curr()), - self._line_col_string())) + raise _NamelistParseError("expected %s but found %r" % + (char_description, str(self._curr()))) def _parse_namelist_group_name(self): r"""Parses and returns a namelist group name at the current position. @@ -1497,11 +1470,11 @@ def _parse_namelist_group_name(self): >>> _NamelistParser('abc')._parse_namelist_group_name() Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: expected '&' but found 'a' at line 1, column 0 + _NamelistParseError: Error in parsing namelist: expected '&' but found 'a' >>> _NamelistParser('&abc')._parse_namelist_group_name() Traceback (most recent call last): ... - _NamelistEOF: Unexpected end of file encountered in namelist. (At line 1, column 4) + _NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser('&abc ')._parse_namelist_group_name() u'abc' >>> _NamelistParser('&abc\n')._parse_namelist_group_name() @@ -1509,15 +1482,15 @@ def _parse_namelist_group_name(self): >>> _NamelistParser('&abc/ ')._parse_namelist_group_name() Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: 'abc/' is not a valid variable name at line 1, column 5 + _NamelistParseError: Error in parsing namelist: 'abc/' is not a valid variable name >>> _NamelistParser('&abc= ')._parse_namelist_group_name() Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: 'abc=' is not a valid variable name at line 1, column 5 + _NamelistParseError: Error in parsing namelist: 'abc=' is not a valid variable name >>> _NamelistParser('& ')._parse_namelist_group_name() Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: '' is not a valid variable name at line 1, column 1 + _NamelistParseError: Error in parsing namelist: '' is not a valid variable name """ self._expect_char("&") self._advance() @@ -1533,7 +1506,7 @@ def _parse_variable_name(self, allow_equals=True): >>> _NamelistParser('abc')._parse_variable_name() Traceback (most recent call last): ... - _NamelistEOF: Unexpected end of file encountered in namelist. (At line 1, column 3) + _NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser('foo(2)= ')._parse_variable_name() u'foo(2)' >>> _NamelistParser('abc ')._parse_variable_name() @@ -1550,20 +1523,29 @@ def _parse_variable_name(self, allow_equals=True): u'abc(1:2:3)' >>> _NamelistParser('abc=')._parse_variable_name() u'abc' + >>> _NamelistParser('abc(1,2) ')._parse_variable_name() + Traceback (most recent call last): + ... + _NamelistParseError: Error in parsing namelist: Multiple dimensions not supported in CIME namelist variables 'abc(1,2)' >>> _NamelistParser('abc, ')._parse_variable_name() Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: 'abc,' is not a valid variable name at line 1, column 4 + _NamelistParseError: Error in parsing namelist: 'abc,' is not a valid variable name >>> _NamelistParser(' ')._parse_variable_name() Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: '' is not a valid variable name at line 1, column 0 + _NamelistParseError: Error in parsing namelist: '' is not a valid variable name """ old_pos = self._pos separators = (' ', '\n', '=') if allow_equals else (' ', '\n') while self._curr() not in separators: self._advance() text = self._text[old_pos:self._pos] + if '(' in text: + expect(')' in text,"Parsing error ") + elif ')' in text: + expect(False,"Parsing error ") + # @ is used in a namelist to put the same namelist variable in multiple groups # in the write phase, all characters in the namelist variable name after # the @ and including the @ should be removed @@ -1575,8 +1557,11 @@ def _parse_variable_name(self, allow_equals=True): text_check = text if not is_valid_fortran_name(text_check): - raise _NamelistParseError("%r is not a valid variable name at %s" % - (str(text), self._line_col_string())) + if re.search(".*\(.*\,.*\)", text_check): + err_str = "Multiple dimensions not supported in CIME namelist variables %r" % str(text) + else: + err_str = "%r is not a valid variable name" % str(text) + raise _NamelistParseError(err_str) name = text.lower() return name @@ -1590,7 +1575,7 @@ def _parse_character_literal(self): >>> _NamelistParser('"abc')._parse_character_literal() Traceback (most recent call last): ... - _NamelistEOF: Unexpected end of file encountered in namelist. (At line 1, column 4) + _NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser('"abc" ')._parse_character_literal() u'"abc"' >>> _NamelistParser("'abc' ")._parse_character_literal() @@ -1598,7 +1583,7 @@ def _parse_character_literal(self): >>> _NamelistParser("*abc* ")._parse_character_literal() Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: *abc* is not a valid character literal at line 1, column 4 + _NamelistParseError: Error in parsing namelist: *abc* is not a valid character literal >>> _NamelistParser("'abc''def' ")._parse_character_literal() u"'abc''def'" >>> _NamelistParser("'abc''' ")._parse_character_literal() @@ -1622,8 +1607,8 @@ def _parse_character_literal(self): break text = self._text[old_pos:self._pos+1] if not is_valid_fortran_namelist_literal("character", text): - raise _NamelistParseError("%s is not a valid character literal at %s" % - (text, self._line_col_string())) + raise _NamelistParseError("%s is not a valid character literal" % + (text)) return text def _parse_complex_literal(self): @@ -1635,21 +1620,21 @@ def _parse_complex_literal(self): >>> _NamelistParser('(1.,2.')._parse_complex_literal() Traceback (most recent call last): ... - _NamelistEOF: Unexpected end of file encountered in namelist. (At line 1, column 6) + _NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser('(1.,2.) ')._parse_complex_literal() u'(1.,2.)' >>> _NamelistParser("(A,B) ")._parse_complex_literal() Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: '(A,B)' is not a valid complex literal at line 1, column 4 + _NamelistParseError: Error in parsing namelist: '(A,B)' is not a valid complex literal """ old_pos = self._pos while self._curr() != ')': self._advance() text = self._text[old_pos:self._pos+1] if not is_valid_fortran_namelist_literal("complex", text): - raise _NamelistParseError("%r is not a valid complex literal at %s" - % (str(text), self._line_col_string())) + raise _NamelistParseError("%r is not a valid complex literal" + % (str(text))) return text def _look_ahead_for_equals(self, pos): @@ -1701,7 +1686,7 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): >>> _NamelistParser('"abc"')._parse_literal() Traceback (most recent call last): ... - _NamelistEOF: Unexpected end of file encountered in namelist. (At line 1, column 5) + _NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser('"abc"')._parse_literal(allow_eof_end=True) u'"abc"' >>> _NamelistParser('(1.,2.) ')._parse_literal() @@ -1709,7 +1694,7 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): >>> _NamelistParser('(1.,2.)')._parse_literal() Traceback (most recent call last): ... - _NamelistEOF: Unexpected end of file encountered in namelist. (At line 1, column 7) + _NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser('(1.,2.)')._parse_literal(allow_eof_end=True) u'(1.,2.)' >>> _NamelistParser('5 ')._parse_literal() @@ -1725,7 +1710,7 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): >>> _NamelistParser('hamburger ')._parse_literal() Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: expected literal value, but got 'hamburger' at line 1, column 9 + _NamelistParseError: Error in parsing namelist: expected literal value, but got 'hamburger' >>> _NamelistParser('5,')._parse_literal() u'5' >>> _NamelistParser('5\n')._parse_literal() @@ -1743,13 +1728,13 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): >>> _NamelistParser('6*')._parse_literal() Traceback (most recent call last): ... - _NamelistEOF: Unexpected end of file encountered in namelist. (At line 1, column 2) + _NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser('6*')._parse_literal(allow_eof_end=True) u'6*' >>> _NamelistParser('foo= ')._parse_literal() Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: expected literal value, but got 'foo=' at line 1, column 4 + _NamelistParseError: Error in parsing namelist: expected literal value, but got 'foo=' >>> _NamelistParser('5,')._parse_literal(allow_name=True) u'5' >>> x = _NamelistParser('foo= ') @@ -1759,7 +1744,7 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): >>> _NamelistParser('6*foo= ')._parse_literal(allow_name=True) Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: expected literal value, but got '6*foo=' at line 1, column 6 + _NamelistParseError: Error in parsing namelist: expected literal value, but got '6*foo=' >>> x = _NamelistParser('foo = ') >>> x._parse_literal(allow_name=True) >>> x._curr() @@ -1799,7 +1784,13 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): if allow_name: separators.append('=') while new_pos != self._len and self._text[new_pos] not in separators: + # allow commas if they are inside () + if self._text[new_pos] == '(': + separators.remove(',') + elif self._text[new_pos] == ')': + separators.append(',') new_pos += 1 + if not allow_eof_end and new_pos == self._len: # At the end of the file, give up by throwing an EOF. self._advance(self._len) @@ -1811,8 +1802,8 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): text = self._text[old_pos:self._pos] if not any(is_valid_fortran_namelist_literal(type_, text) for type_ in ("integer", "logical", "real")): - raise _NamelistParseError("expected literal value, but got %r at %s" - % (str(text), self._line_col_string())) + raise _NamelistParseError("expected literal value, but got %r" + % (str(text))) return text def _expect_separator(self, allow_eof=False): @@ -1855,7 +1846,7 @@ def _expect_separator(self, allow_eof=False): >>> x._expect_separator() Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: expected one of the characters in ' \n,/' but found 'a' at line 1, column 0 + _NamelistParseError: Error in parsing namelist: expected one of the characters in ' \n,/' but found 'a' >>> x = _NamelistParser(" , a") >>> x._expect_separator() True @@ -1888,9 +1879,9 @@ def _expect_separator(self, allow_eof=False): >>> x._expect_separator(allow_eof=True) Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: found group-terminating '/' in file without group names at line 1, column 1 + _NamelistParseError: Error in parsing namelist: found group-terminating '/' in file without group names """ - errstring = "found group-terminating '/' in file without group names at " + errstring = "found group-terminating '/' in file without group names" # Deal with the possibility that we are already at EOF. if allow_eof and self._pos == self._len: return False @@ -1900,8 +1891,7 @@ def _expect_separator(self, allow_eof=False): self._eat_whitespace() if self._curr() == '/': if allow_eof: - raise _NamelistParseError(errstring + - self._line_col_string()) + raise _NamelistParseError(errstring) else: return False except _NamelistEOF: @@ -1940,7 +1930,7 @@ def _parse_name_and_values(self, allow_eof_end=False): >>> _NamelistParser("foo 'bar' /")._parse_name_and_values() Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: expected '=' but found "'" at line 1, column 4 + _NamelistParseError: Error in parsing namelist: expected '=' but found "'" >>> _NamelistParser("foo='bar','bazz' /")._parse_name_and_values() (u'foo', [u"'bar'", u"'bazz'"]) >>> _NamelistParser("foo=,,'bazz',6*/")._parse_name_and_values() @@ -1952,7 +1942,7 @@ def _parse_name_and_values(self, allow_eof_end=False): >>> _NamelistParser("foo= foo2='ban' ")._parse_name_and_values() Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: expected literal value, but got "foo2='ban'" at line 1, column 15 + _NamelistParseError: Error in parsing namelist: expected literal value, but got "foo2='ban'" >>> _NamelistParser("foo=,,'bazz',6* ")._parse_name_and_values(allow_eof_end=True) (u'foo', [u'', u'', u"'bazz'", u'6*']) >>> _NamelistParser("foo(3)='bazz'")._parse_name_and_values(allow_eof_end=True) @@ -1960,7 +1950,7 @@ def _parse_name_and_values(self, allow_eof_end=False): >>> _NamelistParser("foo=")._parse_name_and_values() Traceback (most recent call last): ... - _NamelistEOF: Unexpected end of file encountered in namelist. (At line 1, column 4) + _NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser("foo=")._parse_name_and_values(allow_eof_end=True) (u'foo', [u'']) >>> _NamelistParser("foo= ")._parse_name_and_values(allow_eof_end=True) @@ -1973,6 +1963,7 @@ def _parse_name_and_values(self, allow_eof_end=False): (u'foo', [u'1', u'']) """ name = self._parse_variable_name() + self._eat_whitespace() self._expect_char("=") try: From 6584df0063f155c0148a9d4c8ddbb32b48bcd51d Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 24 Mar 2017 08:42:52 -0600 Subject: [PATCH 17/25] fix ordering issues --- scripts/lib/CIME/XML/namelist_definition.py | 3 ++- scripts/lib/CIME/namelist.py | 20 ++++++++------------ 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/scripts/lib/CIME/XML/namelist_definition.py b/scripts/lib/CIME/XML/namelist_definition.py index 305759c78d6..6e0664a3710 100644 --- a/scripts/lib/CIME/XML/namelist_definition.py +++ b/scripts/lib/CIME/XML/namelist_definition.py @@ -11,6 +11,7 @@ # pylint:disable=wildcard-import,unused-wildcard-import import re +import collections from CIME.namelist import fortran_namelist_base_value, \ is_valid_fortran_namelist_literal, character_literal_to_string, \ @@ -385,7 +386,7 @@ def dict_to_namelist(self, dict_, filename=None): group_name = self.get_group(qualified_varname) expect (group_name is not None, "No group found for var %s"%variable_lc) if group_name not in groups: - groups[group_name] = {} + groups[group_name] = collections.OrderedDict() groups[group_name][variable_lc] = dict_[variable_name] return Namelist(groups) diff --git a/scripts/lib/CIME/namelist.py b/scripts/lib/CIME/namelist.py index 8af9e291fcc..2569bbb8365 100644 --- a/scripts/lib/CIME/namelist.py +++ b/scripts/lib/CIME/namelist.py @@ -101,6 +101,7 @@ # pylint: disable=line-too-long,too-many-lines,invalid-name import re +import collections # Disable these because this is our standard setup # pylint: disable=wildcard-import,unused-wildcard-import @@ -882,14 +883,14 @@ def __init__(self, groups=None): for group_name in groups: expect(group_name is not None, " Got None in groups %s"%groups) group_lc = group_name.lower() - self._groups[group_lc] = {} + self._groups[group_lc] = collections.OrderedDict() for variable_name in groups[group_name]: variable_lc = variable_name.lower() self._groups[group_lc][variable_lc] = \ groups[group_name][variable_name] def clean_groups(self): - self._groups = {} + self._groups = collections.OrderedDict() def get_group_names(self): """Return a list of all groups in the namelist. @@ -998,7 +999,6 @@ def set_variable_value(self, group_name, variable_name, value, var_size=1): """ group_name = group_name.lower() -# print "BEFORE name %s value %s"%(variable_name, value) minindex, maxindex, step = get_fortran_variable_indices(variable_name, var_size) variable_name = get_fortran_name_only(variable_name.lower()) @@ -1016,14 +1016,10 @@ def set_variable_value(self, group_name, variable_name, value, var_size=1): if minindex > tlen: self._groups[group_name][variable_name].extend(['']*(minindex-tlen-1)) - for i in range(minindex-1, maxindex, step): - if i < tlen: - self._groups[group_name][variable_name][i] = value.pop(0) - else: - if tlen < i: - self._groups[group_name][variable_name].extend(['']*(i-tlen)) - self._groups[group_name][variable_name].append(value.pop(0)) - tlen = len(self._groups[group_name][variable_name]) + for i in range(minindex-1, maxindex+step, step): + while len(self._groups[group_name][variable_name]) < i+1: + self._groups[group_name][variable_name].append('') + self._groups[group_name][variable_name][i] = value.pop(0) if len(value) == 0: break @@ -1246,7 +1242,7 @@ def __init__(self, text, groupless=False): # Dictionary with group names as keys, and dictionaries of variable # name-value pairs as values. (Or a single flat dictionary if # `groupless=True`.) - self._settings = {} + self._settings = collections.OrderedDict() # Fortran allows setting a particular index of an array # such as foo(2)='k' # this dict is set to that value if used. From 4bc8257bf18083f7edf5d0a6991b9f82e2feaa4c Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 24 Mar 2017 08:54:23 -0600 Subject: [PATCH 18/25] fix doctests --- scripts/lib/CIME/namelist.py | 40 ++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/scripts/lib/CIME/namelist.py b/scripts/lib/CIME/namelist.py index 2569bbb8365..2179709feab 100644 --- a/scripts/lib/CIME/namelist.py +++ b/scripts/lib/CIME/namelist.py @@ -1993,17 +1993,17 @@ def _parse_namelist_group(self): >>> x = _NamelistParser("&group /") >>> x._parse_namelist_group() >>> x._settings - {u'group': {}} + OrderedDict([(u'group', {})]) >>> x._curr() u'/' >>> x = _NamelistParser("&group\n foo='bar','bazz'\n,, foo2=2*5\n /") >>> x._parse_namelist_group() >>> x._settings - {u'group': {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'5', u'5']}} + OrderedDict([(u'group', {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'5', u'5']})]) >>> x = _NamelistParser("&group\n foo='bar','bazz'\n,, foo2=2*5\n /", groupless=True) >>> x._parse_namelist_group() >>> x._settings - {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'5', u'5']} + OrderedDict([(u'foo', [u"'bar'", u"'bazz'", u'']), (u'foo2', [u'5', u'5'])]) >>> x._curr() u'/' >>> x = _NamelistParser("&group /&group /") @@ -2016,15 +2016,15 @@ def _parse_namelist_group(self): >>> x = _NamelistParser("&group foo='bar', foo='bazz' /") >>> x._parse_namelist_group() >>> x._settings - {u'group': {u'foo': [u"'bazz'"]}} + OrderedDict([(u'group', {u'foo': [u"'bazz'"]})]) >>> x = _NamelistParser("&group foo='bar', foo= /") >>> x._parse_namelist_group() >>> x._settings - {u'group': {u'foo': [u"'bar'"]}} + OrderedDict([(u'group', {u'foo': [u"'bar'"]})]) >>> x = _NamelistParser("&group foo='bar', foo= /", groupless=True) >>> x._parse_namelist_group() >>> x._settings - {u'foo': [u"'bar'"]} + OrderedDict([(u'foo', [u"'bar'"])]) """ group_name = self._parse_namelist_group_name() if not self._groupless: @@ -2056,33 +2056,33 @@ def parse_namelist(self): first by namelist group name, then by variable name. >>> _NamelistParser("").parse_namelist() - {} + OrderedDict() >>> _NamelistParser(" \n!Comment").parse_namelist() - {} + OrderedDict() >>> _NamelistParser(" &group /").parse_namelist() - {u'group': {}} + OrderedDict([(u'group', {})]) >>> _NamelistParser("! Comment \n &group /! Comment\n ").parse_namelist() - {u'group': {}} + OrderedDict([(u'group', {})]) >>> _NamelistParser("! Comment \n &group /! Comment ").parse_namelist() - {u'group': {}} + OrderedDict([(u'group', {})]) >>> _NamelistParser("&group1\n foo='bar','bazz'\n,, foo2=2*5\n / &group2 /").parse_namelist() - {u'group1': {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'5', u'5']}, u'group2': {}} + OrderedDict([(u'group1', {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'5', u'5']}), (u'group2', {})]) >>> _NamelistParser("!blah \n foo='bar','bazz'\n,, foo2=2*5\n ", groupless=True).parse_namelist() - {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'2*5']} + OrderedDict([(u'foo', [u"'bar'", u"'bazz'", u'']), (u'foo2', [u'2*5'])]) >>> _NamelistParser("!blah \n foo='bar','bazz'\n,, foo2=2*5,6\n ", groupless=True).parse_namelist() - {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'2*5', u'6']} + OrderedDict([(u'foo', [u"'bar'", u"'bazz'", u'']), (u'foo2', [u'2*5', u'6'])]) >>> _NamelistParser("!blah \n foo='bar'", groupless=True).parse_namelist() - {u'foo': [u"'bar'"]} + OrderedDict([(u'foo', [u"'bar'"])]) >>> _NamelistParser("foo='bar', foo='bazz'", groupless=True).parse_namelist() - {u'foo': [u"'bazz'"]} + OrderedDict([(u'foo', [u"'bazz'"])]) >>> _NamelistParser("foo='bar', foo=", groupless=True).parse_namelist() - {u'foo': [u"'bar'"]} + OrderedDict([(u'foo', [u"'bar'"])]) >>> _NamelistParser("foo='bar', foo(3)='bazz'", groupless=True).parse_namelist() - {u'foo': [u"'bar'"], u'foo(3)': [u"'bazz'"]} + OrderedDict([(u'foo', [u"'bar'"]), (u'foo(3)', [u"'bazz'"])]) >>> _NamelistParser("foo(2)='bar'", groupless=True).parse_namelist() - {u'foo(2)': [u"'bar'"]} + OrderedDict([(u'foo(2)', [u"'bar'"])]) >>> _NamelistParser("foo(2)='bar', foo(3)='bazz'", groupless=True).parse_namelist() - {u'foo(2)': [u"'bar'"], u'foo(3)': [u"'bazz'"]} + OrderedDict([(u'foo(2)', [u"'bar'"]), (u'foo(3)', [u"'bazz'"])]) """ # Return empty dictionary for empty files. if self._len == 0: From 4ba171196389b8b45b706d161f17811ba2437fcb Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 24 Mar 2017 10:09:38 -0600 Subject: [PATCH 19/25] fix pylint issue --- scripts/lib/CIME/namelist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/lib/CIME/namelist.py b/scripts/lib/CIME/namelist.py index 2179709feab..9614bcad80d 100644 --- a/scripts/lib/CIME/namelist.py +++ b/scripts/lib/CIME/namelist.py @@ -1553,7 +1553,7 @@ def _parse_variable_name(self, allow_equals=True): text_check = text if not is_valid_fortran_name(text_check): - if re.search(".*\(.*\,.*\)", text_check): + if re.search(r".*\(.*\,.*\)", text_check): err_str = "Multiple dimensions not supported in CIME namelist variables %r" % str(text) else: err_str = "%r is not a valid variable name" % str(text) From 00285b737ae2a003df54646fa876246d6305fc01 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 24 Mar 2017 11:48:33 -0600 Subject: [PATCH 20/25] fix spelling in comment --- scripts/lib/CIME/XML/namelist_definition.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/lib/CIME/XML/namelist_definition.py b/scripts/lib/CIME/XML/namelist_definition.py index 6e0664a3710..19b06a49c38 100644 --- a/scripts/lib/CIME/XML/namelist_definition.py +++ b/scripts/lib/CIME/XML/namelist_definition.py @@ -346,7 +346,7 @@ def validate(self, namelist,filename=None): qualified_variable_name = get_fortran_name_only(variable_name) self._expect_variable_in_definition(qualified_variable_name, variable_template) - # Check if can actuax1lly change this variable via filename change + # Check if can actually change this variable via filename change if filename is not None: self._user_modifiable_in_variable_definition(qualified_variable_name) From 454f484df4869e9183643c7081893074f30b9c63 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 24 Mar 2017 11:50:50 -0600 Subject: [PATCH 21/25] remove blank lines --- scripts/lib/CIME/namelist.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/scripts/lib/CIME/namelist.py b/scripts/lib/CIME/namelist.py index 9614bcad80d..e4275353ea1 100644 --- a/scripts/lib/CIME/namelist.py +++ b/scripts/lib/CIME/namelist.py @@ -122,12 +122,6 @@ \))?\s*$""" # end optional index expression , re.IGNORECASE | re.VERBOSE) - - - - - - FORTRAN_LITERAL_REGEXES = {} # Integer literals. _int_re_string = r"(\+|-)?[0-9]+" From 6c9e24950c313cec2926ee4df0193430bc8a2259 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 24 Mar 2017 13:03:54 -0600 Subject: [PATCH 22/25] error for values exceed array size --- scripts/lib/CIME/XML/namelist_definition.py | 2 +- scripts/lib/CIME/namelist.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/lib/CIME/XML/namelist_definition.py b/scripts/lib/CIME/XML/namelist_definition.py index 19b06a49c38..7c6dd8a8755 100644 --- a/scripts/lib/CIME/XML/namelist_definition.py +++ b/scripts/lib/CIME/XML/namelist_definition.py @@ -307,7 +307,7 @@ def is_valid_value(self, name, value): # Check size of input array. if len(expand_literal_list(value)) > size: - return False + expect(False, "Value index exceeds variable size for variable %s"%name) return True def _expect_variable_in_definition(self, name, variable_template): diff --git a/scripts/lib/CIME/namelist.py b/scripts/lib/CIME/namelist.py index e4275353ea1..3a02ec4b891 100644 --- a/scripts/lib/CIME/namelist.py +++ b/scripts/lib/CIME/namelist.py @@ -1949,6 +1949,10 @@ def _parse_name_and_values(self, allow_eof_end=False): (u'foo', [u'2']) >>> _NamelistParser("foo=1,2")._parse_name_and_values(allow_eof_end=True) (u'foo', [u'1', u'2']) + >>> _NamelistParser("foo(1:2)=1,2,3 ")._parse_name_and_values(allow_eof_end=True) + Traceback (most recent call last): + ... + SystemExit: ERROR: Too many values for array foo(1:2) >>> _NamelistParser("foo=1,")._parse_name_and_values(allow_eof_end=True) (u'foo', [u'1', u'']) """ @@ -1976,6 +1980,10 @@ def _parse_name_and_values(self, allow_eof_end=False): break # and if it really is a literal, add it. values.append(literal) + (minindex, maxindex, step) = get_fortran_variable_indices(name) + if minindex > 1 or maxindex > minindex or step > 1: + expect(len(values) <= 1 + ((maxindex - minindex)/step), "Too many values for array %s"%(name)) + return name, values def _parse_namelist_group(self): From 357d31d7a2e24ed13756b03cdda9cfe20c0a7dba Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 24 Mar 2017 13:34:17 -0600 Subject: [PATCH 23/25] fix issues with array limits --- scripts/lib/CIME/namelist.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/scripts/lib/CIME/namelist.py b/scripts/lib/CIME/namelist.py index 3a02ec4b891..d2f1c084d02 100644 --- a/scripts/lib/CIME/namelist.py +++ b/scripts/lib/CIME/namelist.py @@ -215,7 +215,7 @@ def get_fortran_name_only(full_var): m = FORTRAN_NAME_REGEX.search(full_var) return m.group(1) -def get_fortran_variable_indices(varname, varlen=1): +def get_fortran_variable_indices(varname, varlen=1, allow_any_len=False): """ get indices from a fortran namelist variable as a triplet of minindex, maxindex and step >>> get_fortran_variable_indices('foo(3)') @@ -224,6 +224,10 @@ def get_fortran_variable_indices(varname, varlen=1): (1, 2, 3) >>> get_fortran_variable_indices('foo(::)', varlen=4) (1, 4, 1) + >>> get_fortran_variable_indices('foo(::2)', varlen=4) + (1, 4, 2) + >>> get_fortran_variable_indices('foo(::)', allow_any_len=True) + (1, -1, 1) """ m = FORTRAN_NAME_REGEX.search(varname) (minindex, maxindex, step) = (1, varlen, 1) @@ -241,6 +245,11 @@ def get_fortran_variable_indices(varname, varlen=1): if m.group(8) is not None: step = int(m.group(8)) + expect(step > 0 and minindex <= maxindex,"Bad array index values, negative indexing not allowed in %s"%varname) + + if allow_any_len and maxindex == minindex: + maxindex = -1 + return (minindex, maxindex, step) def fortran_namelist_base_value(string): @@ -1980,9 +1989,10 @@ def _parse_name_and_values(self, allow_eof_end=False): break # and if it really is a literal, add it. values.append(literal) - (minindex, maxindex, step) = get_fortran_variable_indices(name) - if minindex > 1 or maxindex > minindex or step > 1: - expect(len(values) <= 1 + ((maxindex - minindex)/step), "Too many values for array %s"%(name)) + (minindex, maxindex, step) = get_fortran_variable_indices(name,allow_any_len=True) + if (minindex > 1 or maxindex > minindex or step > 1) and maxindex > 0: + arraylen =1 + ((maxindex - minindex)/step) + expect(len(values) <= arraylen, "Too many values for array %s"%(name)) return name, values From b284bde4cdd071c070a1db5478835ae021692996 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 24 Mar 2017 13:42:29 -0600 Subject: [PATCH 24/25] remove unneeded restriction on step --- scripts/lib/CIME/namelist.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/lib/CIME/namelist.py b/scripts/lib/CIME/namelist.py index d2f1c084d02..46319c4eacb 100644 --- a/scripts/lib/CIME/namelist.py +++ b/scripts/lib/CIME/namelist.py @@ -245,8 +245,6 @@ def get_fortran_variable_indices(varname, varlen=1, allow_any_len=False): if m.group(8) is not None: step = int(m.group(8)) - expect(step > 0 and minindex <= maxindex,"Bad array index values, negative indexing not allowed in %s"%varname) - if allow_any_len and maxindex == minindex: maxindex = -1 @@ -1991,7 +1989,7 @@ def _parse_name_and_values(self, allow_eof_end=False): values.append(literal) (minindex, maxindex, step) = get_fortran_variable_indices(name,allow_any_len=True) if (minindex > 1 or maxindex > minindex or step > 1) and maxindex > 0: - arraylen =1 + ((maxindex - minindex)/step) + arraylen =max(0,1 + ((maxindex - minindex)/step)) expect(len(values) <= arraylen, "Too many values for array %s"%(name)) return name, values From 75a815a2a55318ea22f9f76afb14384afaed9056 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 24 Mar 2017 15:23:15 -0600 Subject: [PATCH 25/25] one more fix for negative index --- scripts/lib/CIME/XML/namelist_definition.py | 2 +- scripts/lib/CIME/namelist.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/lib/CIME/XML/namelist_definition.py b/scripts/lib/CIME/XML/namelist_definition.py index 7c6dd8a8755..dcb59bd2c85 100644 --- a/scripts/lib/CIME/XML/namelist_definition.py +++ b/scripts/lib/CIME/XML/namelist_definition.py @@ -307,7 +307,7 @@ def is_valid_value(self, name, value): # Check size of input array. if len(expand_literal_list(value)) > size: - expect(False, "Value index exceeds variable size for variable %s"%name) + expect(False, "Value index exceeds variable size for variable %s, allowed array length is %s value array size is %s"%(name, size, len(expand_literal_list(value)))) return True def _expect_variable_in_definition(self, name, variable_template): diff --git a/scripts/lib/CIME/namelist.py b/scripts/lib/CIME/namelist.py index 46319c4eacb..d91a31184c3 100644 --- a/scripts/lib/CIME/namelist.py +++ b/scripts/lib/CIME/namelist.py @@ -248,6 +248,8 @@ def get_fortran_variable_indices(varname, varlen=1, allow_any_len=False): if allow_any_len and maxindex == minindex: maxindex = -1 + expect(step != 0,"Step size 0 not allowed") + return (minindex, maxindex, step) def fortran_namelist_base_value(string): @@ -1017,10 +1019,10 @@ def set_variable_value(self, group_name, variable_name, value, var_size=1): if minindex > tlen: self._groups[group_name][variable_name].extend(['']*(minindex-tlen-1)) - for i in range(minindex-1, maxindex+step, step): - while len(self._groups[group_name][variable_name]) < i+1: + for i in range(minindex, maxindex+2*step, step): + while len(self._groups[group_name][variable_name]) < i: self._groups[group_name][variable_name].append('') - self._groups[group_name][variable_name][i] = value.pop(0) + self._groups[group_name][variable_name][i-1] = value.pop(0) if len(value) == 0: break