Merge remote-tracking branch 'esmci/master' into two_part_system_tests

Want updates particularly for hobart changes

ChangeLog

@@ -1,5 +1,117 @@
 ======================================================================
 
+Originator: Bill Sacks
+Date: Aug 02, 2016
+Tag: cime5.0.5
+Answer Changes: None
+Tests: scripts_regression_tests
+Dependencies:  None (removes dependency of cime5.0.4 on clm branch)
+
+Description:	
+- move LII test from CLM back to CIME (until new API is solidified)
+
+Modified files:
+utils/python/CIME/SystemTests/lii.py
+
+======================================================================
+
+Originator: Jim Edwards
+Date: Aug 01, 2016
+Tag: cime5.0.4
+Answer Changes: None
+Tests: scripts_regression_tests
+Dependencies: Needs clm branch containing LII test
+
+Description:
+
+- Updates of hobart machine 
+- Put provenance back in README -
+- Update TestStatus to run tests multiple times in a case - Fix for
+- Add README files back to test directories
+- Dynamically search for system tests, move LII test to CLM cime_config 
+- Add python profiling tool
+- Minor bug fixes in prep_ocn_mod and prep_wav_mod  pointed out by CRAY compiler	
+
+Modified files:
+
+cime_config/acme/config_files.xml
+cime_config/cesm/config_files.xml
+cime_config/cesm/machines/config_batch.xml
+cime_config/cesm/machines/config_compilers.xml
+cime_config/cesm/machines/config_machines.xml
+driver_cpl/driver/prep_ocn_mod.F90
+driver_cpl/driver/prep_wav_mod.F90
+scripts/Testing/Testcases/README
+scripts/Tools/advanced-py-prof
+scripts/Tools/bless_test_results
+scripts/Tools/case.build
+scripts/Tools/simple-py-prof
+scripts/create_test
+utils/python/CIME/SystemTests/README
+utils/python/CIME/SystemTests/lii.py
+utils/python/CIME/SystemTests/system_tests_common.py
+utils/python/CIME/XML/env_test.py
+utils/python/CIME/case.py
+utils/python/CIME/case_test.py
+utils/python/CIME/check_input_data.py
+utils/python/CIME/system_test.py
+utils/python/CIME/test_scheduler.py
+utils/python/CIME/test_utils.py
+utils/python/CIME/user_mod_support.py
+utils/python/CIME/utils.py
+utils/python/cs.submit.template
+utils/python/tests/scripts_regression_tests.py
+
+
+
+====================================================================
+
+Originator: Jim Edwards
+Date: July 26, 2016
+Tag: cime5.0.3
+Answer Changes: None
+Tests: scripts_regression_tests
+Dependencies:
+
+Brief Summary: 
+- fixes as a result of cesm2_alpha01b testing
+- print output from buildnml
+- improve mismatch argument to NAG compiler
+- fix memleak error if baseline not found
+- add support to create_test for ascii testfile
+- move pecount support from create_test to create_newcase
+- fix LII test, fix ERP test
+- reintroduce --verbose option
+
+User interface changes: none
+
+Modified files: git diff --name-status
+M       ChangeLog
+M       cime_config/acme/config_files.xml
+M       cime_config/cesm/machines/Makefile
+M       cime_config/cesm/machines/config_compilers.xml
+A       cime_config/cesm/machines/nag_mpi_argument.txt
+M       driver_cpl/cime_config/buildexe
+M       scripts/Tools/case.build
+M       scripts/Tools/code_checker
+M       scripts/Tools/component_compare_test.sh
+M       scripts/Tools/preview_namelists
+M       scripts/create_newcase
+M       scripts/create_test
+M       utils/python/CIME/SystemTests/erp.py
+M       utils/python/CIME/SystemTests/lii.py
+M       utils/python/CIME/SystemTests/system_tests_common.py
+M       utils/python/CIME/XML/env_batch.py
+M       utils/python/CIME/build.py
+M       utils/python/CIME/case.py
+M       utils/python/CIME/preview_namelists.py
+M       utils/python/CIME/system_test.py
+M       utils/python/CIME/utils.py
+M       utils/python/tests/scripts_regression_tests.py
+M       utils/python/update_acme_tests.py
+
+======================================================================
+
 Originator: Jim Edwards
 Date: July 22, 2016
 Tag: cime5.0.2

cime_config/acme/config_files.xml

@@ -117,6 +117,23 @@
     <desc>file containing specification of all pe-layouts for primary component (for documentation only - DO NOT EDIT)</desc>
   </entry>
 
+  <entry id="SYSTEM_TESTS_DIR">
+    <type>char</type>
+    <values>
+      <value component="any">$CIMEROOT/utils/python/CIME/SystemTests</value>
+      <value component="clm">$SRCROOT/components/clm/cime_config/SystemTests</value>
+      <value component="cam">$SRCROOT/components/cam/cime_config/SystemTests</value>
+      <value component="pop">$SRCROOT/components/pop/cime_config/SystemTests</value>
+      <value component="cice">$SRCROOT/components/cice/cime_config/SystemTests</value>
+      <value component="cism">$SRCROOT/components/cism/cime_config/SystemTests</value>
+    </values>
+    <group>test</group>
+    <file>env_test.xml</file>
+    <desc>directories containing cime compatible system test modules</desc>
+  </entry>
+
+
+
   <entry id="TESTS_SPEC_FILE">
     <type>char</type>
     <default_value>unset</default_value>

cime_config/cesm/config_files.xml

@@ -113,6 +113,23 @@
     <desc>file containing specification of all pe-layouts for primary component (for documentation only - DO NOT EDIT)</desc>
   </entry>
 
+  <entry id="SYSTEM_TESTS_DIR">
+    <type>char</type>
+    <values>
+      <value component="any">$CIMEROOT/utils/python/CIME/SystemTests</value>
+      <value component="clm">$SRCROOT/components/clm/cime_config/SystemTests</value>
+      <value component="cam">$SRCROOT/components/cam/cime_config/SystemTests</value>
+      <value component="pop">$SRCROOT/components/pop/cime_config/SystemTests</value>
+      <value component="cice">$SRCROOT/components/cice/cime_config/SystemTests</value>
+      <value component="cism">$SRCROOT/components/cism/cime_config/SystemTests</value>
+      <value component="rtm">$SRCROOT/components/rtm/cime_config/SystemTests</value>
+      <value component="mosart">$SRCROOT/components/mosart/cime_config/SystemTests</value>
+    </values>
+    <group>test</group>
+    <file>env_test.xml</file>
+    <desc>directories containing cime compatible system test modules</desc>
+  </entry>
+
   <entry id="TESTS_SPEC_FILE">
     <type>char</type>
     <default_value>unset</default_value>

cime_config/cesm/machines/config_batch.xml

@@ -131,7 +131,6 @@
        <arg flag="--time" name="$JOB_WALLCLOCK_TIME"/>
        <arg flag="-p" name="$JOB_QUEUE"/>
        <arg flag="--account" name="$PROJECT"/>
-       <arg flag="-n" name="$TOTALPES"/>
      </submit_args>
      <directives>
        <directive> --job-name={{ job_id }}</directive>

cime_config/cesm/machines/config_compilers.xml

@@ -843,7 +843,6 @@ for mct, etc.
 </compiler>
 
 <compiler MACH="hobart">
- <MPI_LIB_NAME MPILIB="openmpi"> mpi</MPI_LIB_NAME>
  <MPI_LIB_NAME MPILIB="mvapich2"> mpich</MPI_LIB_NAME>
  <LAPACK_LIBDIR> /usr/lib64 </LAPACK_LIBDIR>
  <ADD_SLIBS>$(shell $(NETCDF_PATH)/bin/nf-config --flibs)</ADD_SLIBS>
@@ -857,6 +856,9 @@ for mct, etc.
  <ADD_LDFLAGS> -Wl,-rpath,$(COMPILER_PATH)/mkl/lib/intel64 </ADD_LDFLAGS>
  <ADD_LDFLAGS> -Wl,-rpath,$(MPI_PATH)/lib</ADD_LDFLAGS>
  <ADD_SLIBS MPILIB="mvapich2"> -mkl=cluster </ADD_SLIBS>
+ <ADD_LDFLAGS> -lifcore</ADD_LDFLAGS>
+ <ADD_CFLAGS> -lifcore</ADD_CFLAGS>
+ <ADD_FFLAGS> -lifcore</ADD_FFLAGS>
  <PFUNIT_PATH>/home/santos/pFUnit/pFUnit_Intel_3_0</PFUNIT_PATH>
   <ADD_FFLAGS MPILIB="mpi-serial"> -mcmodel medium </ADD_FFLAGS>
 </compiler>

cime_config/cesm/machines/config_machines.xml

@@ -369,7 +369,7 @@
 
       <modules compiler="cray">
 	<command name="load">PrgEnv-cray</command>
-	<command name="switch">cce cce/8.5.0</command>
+	<command name="switch">cce cce/8.5.1</command>
       </modules>
       <modules compiler="gnu">
 	<command name="load">PrgEnv-gnu</command>
@@ -468,7 +468,7 @@
     <COMPILERS>intel,gnu,cray</COMPILERS>
     <NODENAME_REGEX>edison</NODENAME_REGEX>
     <MPILIBS>mpt,mpi-serial</MPILIBS>
-    <CESMSCRATCHROOT>$ENV{SCRATCH}</CESMSCRATCHROOT>
+    <CESMSCRATCHROOT>$ENV{CSCRATCH}</CESMSCRATCHROOT>
     <RUNDIR>$CESMSCRATCHROOT/$CASE/run</RUNDIR>
     <EXEROOT>$CESMSCRATCHROOT/$CASE/bld</EXEROOT>
     <DIN_LOC_ROOT>/project/projectdirs/ccsm1/inputdata</DIN_LOC_ROOT>
@@ -534,7 +534,7 @@
       </modules>
       <modules compiler="cray">
 	<command name="load">PrgEnv-cray</command>
-	<command name="switch">cce cce/8.4.3</command>
+	<command name="switch">cce cce/8.5.1</command>
       </modules>
       <modules compiler="gnu">
 	<command name="load">PrgEnv-gnu</command>
@@ -577,10 +577,7 @@
     <DESC>"NCAR CGD Linux Cluster 48 pes/node, batch system is PBS"</DESC>
     <NODENAME_REGEX>hobart</NODENAME_REGEX>
     <COMPILERS>intel,pgi,nag,gnu</COMPILERS>
-    <MPILIBS compiler="nag">openmpi</MPILIBS>
-    <MPILIBS compiler="intel">mvapich2</MPILIBS>
-    <MPILIBS compiler="pgi">mvapich2</MPILIBS>
-    <MPILIBS compiler="gnu">mvapich2</MPILIBS>
+    <MPILIBS>mvapich2</MPILIBS>
     <RUNDIR>/scratch/cluster/$USER/$CASE/run</RUNDIR>
     <EXEROOT>/scratch/cluster/$USER/$CASE/bld</EXEROOT>
     <CESMSCRATCHROOT>/scratch/cluster/$USER</CESMSCRATCHROOT>
@@ -621,22 +618,22 @@
 	<command name="load">compiler/intel/15.0.2.164</command>
       </modules>
       <modules compiler="intel" mpilib="mvapich2">
-	<command name="load">  tool/parallel-netcdf/1.6.1/intel/mvapich2  </command>
+	<command name="load">  tool/parallel-netcdf/1.7.0/intel/mvapich2  </command>
       </modules>
       <modules compiler="pgi">
 	<command name="load">compiler/pgi/15.1</command>
       </modules>
       <modules  compiler="pgi" mpilib="mvapich2">
-	  <command name="load">  tool/parallel-netcdf/1.6.1/pgi/mvapich2  </command>
+	  <command name="load">  tool/parallel-netcdf/1.7.0/pgi/mvapich2  </command>
       </modules>
       <modules compiler="nag">
-	<command name="load">compiler/nag/6.0</command>
+	<command name="load">compiler/nag/6.1</command>
       </modules>
-      <modules compiler="nag" mpilib="openmpi">
-	  <command name="load">  tool/parallel-netcdf/1.6.1/nag/openmpi </command>
+      <modules compiler="nag" mpilib="mvapich2">
+	  <command name="load">  tool/parallel-netcdf/1.7.0/nag/mvapich2 </command>
       </modules>
       <modules compiler="gnu">
-	<command name="load">compiler/gnu/4.8.3</command>
+	<command name="load">compiler/gnu/4.8.5</command>
       </modules>
     </module_system>
     <environment_variables>

driver_cpl/driver/prep_ocn_mod.F90

@@ -1191,8 +1191,8 @@ subroutine prep_ocn_calc_r2x_ox(timer)
 
        call seq_map_map(mapper_Rr2o, r2x_rx, r2x_ox(eri), &
             fldlist='Forr_rofl:Forr_rofi:                       &
-                     Forr_rofl_16O:Forr_rofi_16O:Forr_rofl_18O: &
-                     Forr_rofi_18O:Forr_rofl_HDO:Forr_rofi_HDO', norm=.false.)
+                    & Forr_rofl_16O:Forr_rofi_16O:Forr_rofl_18O: &
+                    & Forr_rofi_18O:Forr_rofl_HDO:Forr_rofi_HDO', norm=.false.)
        if (flood_present) then
           call seq_map_map(mapper_Fr2o, r2x_rx, r2x_ox(eri), &
                fldlist='Flrr_flood', norm=.true.)

driver_cpl/driver/prep_wav_mod.F90

@@ -97,6 +97,7 @@ subroutine prep_wav_init(infodata, atm_c2_wav, ocn_c2_wav, ice_c2_wav)
          wav_present=wav_present      , &
          ocn_gnam=ocn_gnam            , &
          wav_gnam=wav_gnam            , &
+         atm_gnam=atm_gnam            , &
          esmf_map_flag=esmf_map_flag  )
 
     allocate(mapper_sa2w)

scripts/Testing/Testcases/README

@@ -0,0 +1,39 @@
+
+NOTES: 
+Most of the tests have been moved to python in cime/utils/python/CIME/SystemTests
+See the README there for more info.
+
+The remaining tests are:
+
+======================================================================
+    Multi-Instance Tests (smoke)
+======================================================================
+
+NOC    multi-instance validation for single instance ocean (default length)
+       do an initial run test with NINST 2 (other than ocn), with mod to instance 1 (suffix: inst1_base, inst2_mod)
+       do an initial run test with NINST 2 (other than ocn), with mod to instance 2 (suffix: inst1_base, inst2_mod)
+       compare inst1_base with inst2_base 
+       compare inst1_mod  with inst2_mod  
+
+
+======================================================================
+    Performance Tests
+======================================================================
+
+ICP    cice performance test 
+OCP    pop performance test 
+
+======================================================================
+    Archiving Tests
+======================================================================
+
+LAR    long term archive test 
+
+======================================================================
+    Other
+======================================================================
+
+HOMME
+
+
+

scripts/Tools/advanced-py-prof

@@ -0,0 +1,14 @@
+#! /bin/bash -e
+
+if [ "$#" -eq 0 ]; then
+    echo "Usage: advanced-py-prof <python exec and arguments>"
+    exit 0
+fi
+
+# Requires graphviz and gprof2dot
+
+DATE_STAMP=$(date "+%Y-%m-%d_%H%M%S")
+FILE_BASENAME=$(basename $1).${DATE_STAMP}
+python -m cProfile -o ${FILE_BASENAME}.prof "$@"
+gprof2dot -f pstats ${FILE_BASENAME}.prof -o ${FILE_BASENAME}.dot || { echo "This tool requires gprof2dot"; exit 1; }
+dot -Teps ${FILE_BASENAME}.dot -o ${FILE_BASENAME}.eps || { echo "This tool requires graphviz (dot)"; exit 1; }

scripts/Tools/bless_test_results

@@ -13,7 +13,7 @@ with versions of the files that you should not bless.
 from standard_script_setup import *
 
 import wait_for_tests, compare_namelists, simple_compare
-from CIME.system_test import NAMELIST_PHASE
+from CIME.test_scheduler import NAMELIST_PHASE
 from CIME.utils import run_cmd, run_cmd_no_fail, expect
 from CIME.XML.machines import Machines
 

scripts/Tools/case.build

@@ -5,27 +5,10 @@ Script to build a case.
 """
 
 from standard_script_setup import *
-from CIME.SystemTests.eri import ERI
-from CIME.SystemTests.erp import ERP
-from CIME.SystemTests.err import ERR
-from CIME.SystemTests.ers import ERS
-from CIME.SystemTests.lii import LII
-from CIME.SystemTests.nck import NCK
-from CIME.SystemTests.ncr import NCR
-from CIME.SystemTests.noc import NOC
-from CIME.SystemTests.pea import PEA
-from CIME.SystemTests.pem import PEM
-from CIME.SystemTests.pet import PET
-from CIME.SystemTests.pfs import PFS
-from CIME.SystemTests.rep import REP
-from CIME.SystemTests.seq import SEQ
-from CIME.SystemTests.sms import SMS
-from CIME.SystemTests.ssp import SSP
-from CIME.SystemTests.system_tests_common import *
 
 import CIME.build as build
 from CIME.case import Case
-from CIME.utils           import expect, append_status
+from CIME.utils           import expect, append_status, find_system_test
 from CIME.XML.files       import Files
 from CIME.XML.component   import Component
 
@@ -101,10 +84,8 @@ def _main_func(description):
         elif(testname is not None):
             logging.warn("Building test for %s in directory %s" %
                          (testname, caseroot))
-            try:
-                test = globals()[testname](case)
-            except KeyError:
-                expect(False, "Could not find a test called '%s'" % testname)
+            test = find_system_test(testname, case)(case)
+
             append_status("case.testbuild starting ",
                          caseroot=caseroot,sfile="CaseStatus")
             test.build(sharedlib_only=sharedlib_only, model_only=model_only)

scripts/Tools/code_checker

@@ -88,9 +88,11 @@ def check_code(dir_to_check, num_procs, files):
         os.environ["PYTHONPATH"] = dir_to_check
 
     # Get list of files to check
-    files_to_check = run_cmd_no_fail('find %s -name "*.py"' % dir_to_check).splitlines()
+    files_to_check = run_cmd_no_fail('git ls-files %s' % dir_to_check).splitlines()
     if files:
         files_to_check = [item for item in files_to_check if matches(item, files)]
+    else:
+        files_to_check = [item for item in files_to_check if item.endswith(".py")]
 
     pool = ThreadPool(num_procs)
     results = pool.map(run_pylint, files_to_check)