Merge pull request #386 from PCMDI/343_pjg_simple_demos

343 pjg simple demos

.travis.yml

@@ -6,12 +6,14 @@ language: python
 
 before_install:
   - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get update -qq; sudo apt-get install -y gfortran ; fi
+  - export UVCDAT_FEATURE=""
+  - if [ "$TRAVIS_OS_NAME" = "linux" ]; then export UVCDAT_FEATURE="-nox"; fi
   - if [ "$TRAVIS_OS_NAME" = "linux" ] && ! [ -d ${HOME}/miniconda] ; then wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh; fi
 #  - if [ "$TRAVIS_OS_NAME" = "osx" ]; then brew update ; fi 
 #  - if [ "$TRAVIS_OS_NAME" = "osx" ] && ! [ -d ${HOME}/miniconda ]; then wget https://repo.continuum.io/miniconda/Miniconda-latest-MacOSX-x86_64.sh -O miniconda.sh; fi
   - if [ "$TRAVIS_OS_NAME" = "osx" ] && ! [ -d /usr/local/gfortran ]; then wget https://github.com/UV-CDAT/uvcdat/releases/download/v2.4.1/gfortran-4.9.2-Mac.tar.gz -O ~/gfortran-4.9.2-Mac.tar.gz ; pushd / ; sudo tar xzvf ~/gfortran-4.9.2-Mac.tar.gz ; pushd ; fi
   - export PATH="$HOME/miniconda/bin:$PATH"
-  - if [ ! -d ${HOME}/miniconda ]; then bash miniconda.sh -b -p $HOME/miniconda ; conda install --yes --channel uvcdat/label/nightly -c uvcdat uvcdat-nox  ; fi
+  - if [ ! -d ${HOME}/miniconda ]; then bash miniconda.sh -b -p $HOME/miniconda ; conda install --yes --channel uvcdat/label/nightly -c uvcdat uvcdat${UVCDAT_FEATURE} hdf5==1.8.16  ; fi
   - hash -r
   - conda config --set always_yes yes --set changeps1 no
   - conda update -q conda
@@ -23,4 +25,5 @@ install:
   - python setup.py install
 
 script:
+    - export HDF5_DISABLE_VERSION_CHECK=1
     - python test/test_suite.py # --no-graphics

demo/parameter_files/pmp_input_parameters_demo1.py

@@ -1,49 +1,73 @@
 ################################################################################
-#  SAMPLE INPUT PARAMETER FILE FOR THE PCMDI METRICS PACKAGE (PMP V1.1)
-#  THIS IS A VERY SIMPLE EXAMPLE, ONLY COMPUTING STATISTICS FOR ONE MODEL VERSION AND ONE VARIABLE 
-#  SAMPLE MODEL AND OBSERVATIONAL DATA TO RUN THIS CODE ARE AVAILABLE FROM: 
-# 
+#  THIS IS A SIMPLE EXAMPLE mean climate "parameter file" USED IN A DEMO EXECUTION OF THE PMP
+
+#  For more info, see https://github.com/PCMDI/pcmdi_metrics/wiki/Using-the-package
+#
+# THIS PARAMETER FILE IS USED TO COMPUTE STATISTICS FOR ONE MODEL VERSION AND ONE VARIABLE ONLY
+
 # NOTES FOR PYTHON NEWBIES: ON ANY GIVEN LINE, ANTHING TO THE RIGHT OF A "#" IS CONSIDERED A COMMENT IN PYTHON  
 # IN THIS SIMPLE EXAMPLE WE DEFINE CHARACTER STRINGS, BUT ALSO ONE OF THE MOST BASIC AND POWERFUL PYTHON OBJECTS KNOWN AS A LIST.  
 # PYTHON LISTS ARE DEFINED WITH SQUARE BRACKETS [] ... FOR MORE INFO SEE: https://docs.python.org/2/tutorial/datastructures.html
 ################################################################################
+#
+# First, we define the 'template' used by the PMP to construct file names and paths that correspond to the location of the model and observatinal data.
+# Keywords in between %() will be automatically filled by PMP
+# In this example we only use four of the 'official' keys: 'variable', 'model_version', 'realization' and 'period'
+# some of these are defined later in the parameter file
+# For a complete list of official keys see: 
+filename_template = "%(variable)_%(model_version)_Amon_amip_%(realization)_%(period)-clim.nc"
+# First lets define an case id to help us differentiate between many se of parameter files
+# The python 'case_id' variable is optional in the parameter file
+# non the less it is still a 'reserved' variable for the PMP
+case_id = 'simple-test1'
+# Now we will defined the list of models to use
+# For this we use the 'reserved' python variable: model_versions
+# THIS IS A MANDATORY ENTRY
+# This is our first list of a python list, in this case there is only one entry
+model_versions = ['ACCESS1-0']   
 
-## FIRST USE OF A PYTHON LIST, IN THIS CASE IT HAS ONLY ONE ENTRY 
-model_versions = ['ACCESS1-0']   # THIS IS A MANDETORY ENTRY FOR DOCUMENTING RESULTS
-
-###############################################################################
-## DATA LOCATION: MODELS, OBS AND METRICS OUTPUT
-## ROOT PATH FOR MODELS CLIMATOLOGIES
-#mod_data_path = '/work/metricspackage/mod_clims/cmip5-amip'
-mod_data_path = './demo_obs_model_data/mods/'
-filename_template = "pr_Amon_ACCESS1-0_amip_r1i1p1_197901-198912-clim-ac.nc"
+# The following lines will tell PMP where the data reside on your system
+# Note that we could use the 'templating' filename system here as well
+# Also note that these path are 'relative' to our current working path
+# But one could use absolute paths as well
+## MODELS DATA LOCATION
+mod_data_path = 'pmp_demo/cmip5clims_metrics_package-amip/'
 ## ROOT PATH FOR OBSERVATIONS
-obs_data_path = './demo_obs_model_data/obs/'
+obs_data_path = 'pmp_demo/obs/'
 
 ## DIRECTORY WHERE TO SAVE RESULTS
-case_id = 'simple-test1'
-metrics_output_path = './pmp-test/'  # USER CHOOSES, RESULTS STORED IN  metrics_output_path + case_id   
-###############################################################################
+# USER CHOOSES, RESULTS STORED IN  metrics_output_path + case_id   
+metrics_output_path = './pmp_demo/%(case_id)/'  
 
 # OBSERVATIONS TO USE: CHOICES INCLUDE 'default','alternate1','alternate2',... AND ARE VARIABLE DEPENDENT 
 ref = ['default']  #,'alternate1','alternate2']
 
 ## A PYTHON LIST OF VARIABLES TO COMPUTE STATISTICS
-vars = ['pr']  # THIS EXAMPLE ONLY INCLUDES ONE FIELD, PRECIPICATION
+# THIS EXAMPLE ONLY INCLUDES ONE FIELD, PRECIPICATION
+vars = ['pr']  
 
-# INTERPOLATION OPTIONS
-targetGrid        = '2.5x2.5' # OPTIONS: '2.5x2.5' or an actual cdms2 grid object
-regrid_tool       = 'esmf' #'regrid2' # OPTIONS: 'regrid2','esmf'
-regrid_method     = 'linear'  # OPTIONS: 'linear','conservative', only if tool is esmf
+# INTERPOLATION (REGRIDDING) OPTIONS
+# First our target grid, i.e the final grid onto which both model and obs will be put
+# OPTIONS: '2.5x2.5' or an actual cdms2 grid object
+targetGrid        = '2.5x2.5' 
+# Now let's select which cdms2 regrid tool we will use
+regrid_tool       = 'esmf' 
+# Some regrid tools also require to specify which method of regriding to use
+# OPTIONS: 'linear','conservative', only if tool is esmf
+regrid_method     = 'linear'
 
 # SIMULATION PARAMETERS (required in PMP v1.1)
-period = '1979-1989'  # PERIOD OF CLIMATOLOGY
-realization = 'r1i1p1' # REALIZATION
-
-
-
-
+# These are manadatory in any PMP parameter file
+# Beside remember that these are actually using in our templating system defined above
+# PERIOD OF CLIMATOLOGY
+period = '198001-200512'
+# MODEL REALIZATION
+realization = 'r1i1p1'
 
+### DONE!
 
+# THIS DEMO PARAMETER FILE IS AVAILABLE AT: https://github.com/PCMDI/pcmdi_metrics/blob/master/demo/parameter_files/pmp_input_parameters_demo1.py  
+# FOR A COMPLETE LISTING OF ALL AVAILABLE KEYS IN THE PMP PARAMETER FILES SEE: https://github.com/PCMDI/pcmdi_metrics/blob/master/doc/parameter_files/input_parameter_file_all_options.py
 
+
 

demo/pmp_demo.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python
+
+# This is a very simple demo for new users
+import subprocess
+import requests
+import hashlib
+import os
+import sys
+import shlex
+import genutil
+
+
+class bgcolor:
+    HEADER = '\033[95m'
+    OKBLUE = '\033[94m'
+    OKGREEN = '\033[92m'
+    WARNING = '\033[93m'
+    FAIL = '\033[91m'
+    ENDC = '\033[0m'
+    BOLD = '\033[1m'
+    UNDERLINE = '\033[4m'
+
+
+def comment(text,cont="[Press enter]"):
+    print text
+    if cont is None:
+        return
+    else:
+        return raw_input(cont)
+
+def describe(demo_file,colorized=True):
+    f = open(demo_file)
+    for l in f.xreadlines(): # goes thru the file
+        ln = l.strip()
+        if ln=="":
+            continue
+        if ln[0]=="#":  # comment
+            while ln[0]=="#":
+                ln=ln[1:]
+                if ln=="":
+                    break
+            if colorized:
+                bc = bgcolor.OKBLUE
+            else:
+                bc = ""
+            comment(bc+ln.strip()+bgcolor.ENDC,None)
+        else:
+            if colorized:
+                bc = bgcolor.BOLD+bgcolor.UNDERLINE+bgcolor.HEADER
+            else:
+                bc = ""
+            print
+            comment(bc+ln+bgcolor.ENDC)
+            print
+    if colorized:
+        bc = bgcolor.OKBLUE
+    else:
+        bc = ""
+    comment(bc+"This ends this parameter file"+bgcolor.ENDC)
+
+def demo(demo_file,title,colorized=True):
+    comment("""
+        PMP Demo: %s 
+
+    This is a demonstration of the PMP
+    It will download some observation and model data for you
+    It will then demonstrate how to setup a parameter file to execute PMP on these
+    It will run the PMP 
+    It will show you where to find the results and how to look at them""" % title)
+
+    cont = comment("""We will now download and untar a small set of data for the demo
+    Data will be untarred in the 'pmp_demo' directory created in the current directory""","Continue? [Y/n]")
+    if cont.strip().lower() not in ["","y","yes"]:
+        sys.exit()
+
+    ## Download data
+    demo_pth = os.path.join(os.getcwd(),"pmp_demo")
+    if not os.path.exists(demo_pth):
+        os.makedirs(demo_pth)
+    #  http://oceanonly.llnl.gov/gleckler1/pmp-demo-data/pmpv1.1_demodata.tar
+    tar_filename = "pmpv1.1_demodata.tar"
+    tar_pth = os.path.join(demo_pth,tar_filename)
+
+    good_md5 = "a6ef8f15457378ff36fd46e8fbf5f157"
+
+    attempts = 0
+    while attempts < 3:
+        md5 = hashlib.md5()
+        if os.path.exists(tar_filename):
+            f = open(tar_filename)
+            md5.update(f.read())
+            if md5.hexdigest() == good_md5:
+                attempts = 5
+                continue
+        print "Downloading: ", tar_filename
+        r = requests.get("http://oceanonly.llnl.gov/gleckler1/pmp-demo-data/pmpv1.1_demodata.tar", stream=True)
+        with open(tar_pth, 'wb') as f:
+            for chunk in r.iter_content(chunk_size=1024):
+                if chunk:  # filter local_filename keep-alive new chunks
+                    f.write(chunk)
+                    md5.update(chunk)
+        f.close()
+        if md5.hexdigest() == good_md5:
+            attempts = 5
+        else:
+            attempts += 1
+
+    comment("Successfuly downloaded demo tarball\nNow untarring it", None)
+
+    tar_process = subprocess.Popen(shlex.split("tar xvf %s"%tar_pth),cwd=demo_pth)
+    tar_process.wait()
+
+    comment("Success! Files are now untarred in %s\nLet's run this demo!\n" % demo_pth,None)
+    comment("""The PMP package runs off a 'parameter' file which needs to be edited by the user
+    Please kindly take a look at our sample parameter file in: %s""" % demo_file)
+
+    describe(demo_file)
+    cmd = "pcmdi_metrics_driver.py -p %s" % demo_file
+    comment("We will now run the pmp using this parameter file\nTo do so we are using the follwoing command\n%s" % cmd)
+    pmp = subprocess.Popen(shlex.split(cmd))
+    sys.path.insert(0,os.path.dirname(demo_file))
+    exec("import %s as pmp_param" % os.path.basename(demo_file)[:-3])
+    pmp.wait()
+    loc = genutil.StringConstructor(os.path.join(pmp_param.metrics_output_path))
+    for att in ["case_id","model_version","period","realization","period"]:
+        if hasattr(pmp_param,att):
+            setattr(loc,att,getattr(pmp_param,att))
+    comment("You can now look at the results in: %s%s%s" % (bgcolor.HEADER+bgcolor.BOLD,loc(),bgcolor.ENDC))
+

demo/pmp_demo_1.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+import sys
+import os
+
+bd,nm=os.path.split(sys.argv[0])
+print bd,nm
+sys.path.insert(0,bd)
+import pmp_demo
+pmp_demo.demo(os.path.join(sys.prefix,"share","pmp","demo","pmp_input_parameters_demo1.py"),"simple-test1")