Add a utility class to copy and modify user_nl_files in system tests

Test suite: None
Test baseline: N/A
Test namelist changes: N/A
Test status: N/A

Ran unit tests with:
[roo2:~/cime/utils/python]$ python -m unittest discover
[roo2:~/cime/utils/python]$ python3 -m unittest discover

Fixes: None

User interface changes?: No

Code review: None

utils/python/CIME/SystemTests/test_utils/tests/test_user_nl_copier.py

@@ -0,0 +1,227 @@
+#!/usr/bin/env python
+
+import unittest
+import os
+import shutil
+import tempfile
+from CIME.SystemTests.test_utils.user_nl_copier import UserNLCopier
+
+class TestUserNLCopier(unittest.TestCase):
+
+    # ========================================================================
+    # Test helper functions
+    # ========================================================================
+
+    def setUp(self):
+        self._casedir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(self._casedir, ignore_errors=True)
+
+    def write_user_nl_file(self, component, contents, suffix=''):
+        """Write contents to a user_nl file in the case directory. Returns the
+        basename (i.e., not the full path) of the file that is created.
+
+        For a component foo, with the default suffix of '', the file name will
+        be user_nl_foo
+
+        If the suffix is '_0001', the file name will be user_nl_foo_0001
+        """
+
+        filename = 'user_nl_' + component + suffix
+
+        with open(os.path.join(self._casedir, filename), 'w') as user_nl_file:
+            user_nl_file.write(contents)
+
+        return filename
+
+    def assertFileContentsEqual(self, expected, filepath, msg=None):
+        """Asserts that the contents of the file given by 'filepath' are equal to
+        the string given by 'expected'. 'msg' gives an optional message to be
+        printed if the assertion fails."""
+
+        with open(filepath, 'r') as myfile:
+            contents = myfile.read()
+
+        self.assertEqual(expected, contents, msg=msg)
+
+    # ========================================================================
+    # Begin actual tests
+    # ========================================================================
+
+    def test_save(self):
+        # Define some variables
+        component = 'foo'
+        save_dirname = 'saved_files'
+        orig_contents = 'bar = 42\n'
+
+        # Setup
+        filename = self.write_user_nl_file(component, orig_contents)
+        nl_copier = UserNLCopier(casedir=self._casedir, component=component,
+                                 save_dirname=save_dirname)
+
+        # Exercise
+        nl_copier.save_user_nl_files()
+
+        # Verify
+        self.assertTrue(os.path.isfile(
+            os.path.join(self._casedir, save_dirname, filename)),
+            msg = 'copied file should exist in save directory')
+
+    def test_save_with_no_file_raises_exception(self):
+        # Setup
+        nl_copier = UserNLCopier(casedir=self._casedir, component='foo')
+
+        # Exercise & verify
+        self.assertRaisesRegexp(RuntimeError, "No user_nl files found",
+                                nl_copier.save_user_nl_files)
+
+    def test_save_called_twice_raises_exception(self):
+        # Define some variables
+        component = 'foo'
+        orig_contents = 'bar = 42\n'
+
+        # Setup
+        with open(os.path.join(self._casedir, 'user_nl_' + component), 'w') as user_nl_file:
+            user_nl_file.write(orig_contents)
+
+        nl_copier = UserNLCopier(casedir=self._casedir, component=component)
+
+        # Exercise & Verify
+        nl_copier.save_user_nl_files()
+        self.assertRaises(RuntimeError, 
+                          nl_copier.save_user_nl_files)
+
+    def test_save_with_existing_directory(self):
+        # It should be okay to call save with an existing directory.
+
+        # Define some variables
+        component = 'foo'
+        save_dirname = 'saved_files'
+        orig_contents = 'bar = 42\n'
+
+        # Setup
+        filename = self.write_user_nl_file(component, orig_contents)
+        os.makedirs(os.path.join(self._casedir, save_dirname))
+        nl_copier = UserNLCopier(casedir=self._casedir, component=component,
+                                 save_dirname=save_dirname)
+
+        # Exercise
+        nl_copier.save_user_nl_files()
+
+        # Verify
+        self.assertTrue(os.path.isfile(
+            os.path.join(self._casedir, save_dirname, filename)),
+            msg = 'copied file should exist in save directory')
+
+    def test_save_overwriting_existing_files_raises_exception(self):
+        # Trying to call save in a way that would overwrite existing files
+        # should raise an exception
+
+        # Define some variables
+        component = 'foo'
+        save_dirname = 'saved_files'
+        orig_contents = 'bar = 42\n'
+
+        # Setup
+        filename = self.write_user_nl_file(component, orig_contents)
+        os.makedirs(os.path.join(self._casedir, save_dirname))
+        shutil.copy(os.path.join(self._casedir, filename),
+                    os.path.join(self._casedir, save_dirname, filename))
+        nl_copier = UserNLCopier(casedir=self._casedir, component=component,
+                                 save_dirname=save_dirname)
+
+        # Exercise & verify
+        self.assertRaisesRegexp(RuntimeError,
+                                "Attempt to overwrite existing saved files",
+                                nl_copier.save_user_nl_files)
+
+    def test_save_multiple_files(self):
+        # Define some variables
+        component = 'foo'
+        save_dirname = 'saved_files'
+        orig_contents1 = 'bar = 42\n'
+        orig_contents2 = 'bar = 17\n'
+
+        # Setup
+        filename1 = self.write_user_nl_file(component, orig_contents1, suffix='_0001')
+        filename2 = self.write_user_nl_file(component, orig_contents2, suffix='_0002')
+
+        nl_copier = UserNLCopier(casedir=self._casedir, component=component,
+                                 save_dirname=save_dirname)
+
+        # Exercise
+        nl_copier.save_user_nl_files()
+
+        # Verify
+        self.assertTrue(os.path.isfile(
+            os.path.join(self._casedir, save_dirname, filename1)),
+            msg = 'copied file 1 should exist in save directory')
+        self.assertTrue(os.path.isfile(
+            os.path.join(self._casedir, save_dirname, filename2)),
+            msg = 'copied file 2 should exist in save directory')
+
+
+    def test_append(self):
+        # Define some variables
+        component = 'foo'
+        # deliberately exclude new line from file contents, to make sure that's
+        # handled correctly
+        orig_contents = 'bar = 42'
+        contents_to_append = 'baz = 101'
+
+        # Setup
+        filename = self.write_user_nl_file(component, orig_contents)
+        nl_copier = UserNLCopier(casedir=self._casedir, component=component)
+        nl_copier.save_user_nl_files()
+
+        # Exercise
+        nl_copier.append_to_saved_files(contents_to_append)
+
+        # Verify
+        expected_contents = orig_contents + '\n' + contents_to_append + '\n'
+        self.assertFileContentsEqual(expected_contents,
+                                     os.path.join(self._casedir, filename))
+
+
+    def test_two_appends(self):
+        # If you call append twice, only the second append should be present in
+        # the final file.
+        #
+        # This test also tests appending with multi-instance
+
+        # Define some variables
+        component = 'foo'
+        orig_contents1 = 'bar = 42'
+        orig_contents2 = 'bar = 17'
+        contents_to_append_first = 'baz = 101'
+        contents_to_append_second = 'baz = 201'
+
+        # Setup
+        filename1 = self.write_user_nl_file(component, orig_contents1, suffix='_0001')
+        filename2 = self.write_user_nl_file(component, orig_contents2, suffix='_0002')
+        nl_copier = UserNLCopier(casedir=self._casedir, component=component)
+        nl_copier.save_user_nl_files()
+
+        # First append should not affect final result
+        nl_copier.append_to_saved_files(contents_to_append_first)
+
+        # Exercise
+        nl_copier.append_to_saved_files(contents_to_append_second)
+
+        # Verify
+        expected_contents1 = orig_contents1 + '\n' + contents_to_append_second + '\n'
+        expected_contents2 = orig_contents2 + '\n' + contents_to_append_second + '\n'
+        self.assertFileContentsEqual(expected_contents1,
+                                     os.path.join(self._casedir, filename1))
+        self.assertFileContentsEqual(expected_contents2,
+                                     os.path.join(self._casedir, filename2))
+
+    def test_append_without_save_raises_exception(self):
+        # Setup
+        nl_copier = UserNLCopier(casedir=self._casedir, component='foo')
+
+        # Exercise & verify
+        self.assertRaisesRegexp(RuntimeError,
+                                "save_user_nl_files must be called before append_to_saved_files",
+                                nl_copier.append_to_saved_files, 'bar')

utils/python/CIME/SystemTests/test_utils/user_nl_copier.py

@@ -0,0 +1,124 @@
+import shutil
+import os
+import glob
+
+class UserNLCopier(object):
+    """This class can be used to save the original user_nl files for a given
+    component, then restore them with some appended modifications for each run
+    of a test.
+
+    Usage is:
+
+    In the constructor for a test:
+    self._nl_copier = UserNLCopier(casedir, component)
+
+    Then, after the user_nl files have been copied into the case directory:
+    self._nl_copier.save_user_nl_files()
+
+    Then, in the initial setup for each run:
+    self._nl_copier.append_to_saved_files(string_to_append)
+
+
+    If you need to handle user_nl files for multiple components in your test, you should
+    create a separate UserNLCopier object for each component (e.g., one for clm,
+    one for cam, etc.)
+    """
+
+    def __init__(self, casedir, component, save_dirname = 'user_nl_orig'):
+        """Creates a UserNLCopier object
+
+        casedir: full path of the case directory
+
+        component: name of component (e.g., 'clm'). This is used to determine
+        which user_nl files are copied and later modified. For example, for
+        component='clm', this object will operate on all user_nl files matching
+        the pattern 'user_nl_clm*'. (We do a wildcard match to handle
+        multi-instance tests.)
+
+        save_dirname: name of directory to be created within the case directory,
+        containing saved copies of the relevant user_nl file(s)
+        """
+
+        self._casedir = casedir
+        self._component = component
+        self._save_dirname = save_dirname
+        self._save_fullpath = os.path.join(self._casedir, self._save_dirname)
+        self._files_saved = False
+
+    def save_user_nl_files(self):
+        """Save original user_nl files so that these originals can be restored
+        later. This should be called exactly once per test, after the user_nl
+        files have been copied into the case directory."""
+
+        if self._files_saved:
+            raise RuntimeError('Attempt to call save_user_nl_files twice')
+
+        if not os.path.exists(self._save_fullpath):
+            os.makedirs(self._save_fullpath)
+
+        files = self._get_list_of_user_nl_files(self._casedir)
+
+        if len(files) == 0:
+            raise RuntimeError('No user_nl files found for component ' + self._component)
+
+        for one_file in files:
+            orig_file = os.path.join(self._casedir, one_file)
+            saved_file = os.path.join(self._save_fullpath, one_file)
+            if (os.path.exists(saved_file)):
+                raise RuntimeError('Attempt to overwrite existing saved files in ' +
+                                   self._save_fullpath)
+            shutil.copy(orig_file, saved_file)
+
+        self._files_saved = True
+
+    def append_to_saved_files(self, contents):
+        """Copy the saved files back to the case directory, then append the
+        string given by 'contents' to the end of each of the saved
+        user_nl files for the given component (there may be multiple such user_nl files in
+        the case of a multi-instance test).
+
+        Also puts new lines before and after the appended text - so 'contents'
+        does not need to contain a trailing new line (but it's also okay if it
+        does).
+
+        Because this method starts with the saved version of the files (from the
+        call to save_user_nl_files), you can NOT use this twice to append
+        additional text: the second call will overwrite the contents added in
+        the first call.
+
+        This must be called after save_user_nl_files.
+
+        contents: string giving the contents to append to the end of each user_nl file
+        """
+
+        if not self._files_saved:
+            raise RuntimeError('save_user_nl_files must be called before append_to_saved_files')
+
+        files = self._get_list_of_user_nl_files(self._save_fullpath)
+
+        if len(files) == 0:
+            raise RuntimeError('No user_nl files found for component ' + self._component)
+
+        for one_file in files:
+            saved_file = os.path.join(self._save_fullpath, one_file)
+            new_file = os.path.join(self._casedir, one_file)
+            shutil.copy(saved_file, new_file)
+            with open(new_file, 'a') as user_nl_file:
+                user_nl_file.write('\n' + contents + '\n')
+
+
+    def _get_list_of_user_nl_files(self, path):
+        """Get a list of all user_nl files in the current path for the component
+        of interest. For a component 'foo', we match all files of the form
+        user_nl_foo* - with a wildcard match at the end in order to match files
+        in a multi-instance case.
+
+        The list of returned files gives just the file names themselves (i.e.,
+        the basenames).
+        """
+
+        file_pattern = 'user_nl_' + self._component + '*'
+        file_list = glob.glob(os.path.join(self._casedir, file_pattern))
+        file_basename_list = [os.path.basename(one_file) for one_file in file_list]
+
+        return file_basename_list