Added ImageDiffer as testers + increased tolerance for a user_guide t…

…est (image) for MASTER to pass. Close #1068

rook/DiffUtils.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """
-Utilities for comparing strings.
+  Utilities for comparing strings.
 """
 import re
 import sys

scripts/TestHarness/testers/RAVENImageDiff.py

@@ -15,7 +15,7 @@
 This tests images against a expected image.
 """
 from __future__ import division, print_function, unicode_literals, absolute_import
-import os
+import os, sys
 
 try:
   from scipy.misc import imread
@@ -24,26 +24,29 @@
   import scipy
   correctImport = False
 
-import DiffUtils as DU
+from Tester import Differ
 
 class ImageDiff:
   """
     ImageDiff is used for comparing two image files.
   """
 
-  def __init__(self, test_dir, out_file, **kwargs):
+  def __init__(self, out_files, gold_files, relative_error=1e-10, zero_threshold=None):
     """
       Create an ImageDiff class
-      @ In, test_dir, string, the directory where the test takes place
-      @ In, out_file, the files to be compared.
-         They will be in test_dir + out_file and test_dir + gold + out_file
-      @ In, args, other arguments that may be included:
+      @ In, out_files, the files to be compared.
+      @ In, gold_files, the files to be compared to the out_files.
+      @ In, relative_error, float, optional, relative error
+      @ In, zero_threshold, float, optional, if a number <= abs(zero_threshold) it will be considered 0
+      @ Out, None.
     """
-    self.__out_file = out_file
-    self.__messages = ""
+    #assert len(out_files) == len(gold_files)
+    self.__out_files = out_files
+    self.__gold_files = gold_files
+    self.__message = ""
     self.__same = True
-    self.__test_dir = test_dir
-    self.__options = kwargs
+    self.__rel_err = relative_error
+    self.__zero_threshold = float(zero_threshold) if zero_threshold is not None else 0.0
 
   def diff(self):
     """
@@ -56,60 +59,111 @@ def diff(self):
     """
     # read in files
     files_read = False
-    for outfile in self.__out_file:
-      test_filename = os.path.join(self.__test_dir, outfile)
-      gold_filename = os.path.join(self.__test_dir, 'gold', outfile)
+    for test_filename, gold_filename in zip(self.__out_files, self.__gold_files):
+      #test_filename = os.path.join(self.__test_dir, outfile)
+      #gold_filename = os.path.join(self.__test_dir, 'gold', outfile)
       if not os.path.exists(test_filename):
         self.__same = False
-        self.__messages += 'Test file does not exist: '+test_filename
+        self.self.__message += 'Test file does not exist: '+test_filename
       elif not os.path.exists(gold_filename):
         self.__same = False
-        self.__messages += 'Gold file does not exist: '+gold_filename
+        self.self.__message += 'Gold file does not exist: '+gold_filename
       else:
         files_read = True
     #read in files
     if files_read:
       if not correctImport:
-        self.__messages += 'ImageDiff cannot run with scipy version less '+\
+        self.self.__message += 'ImageDiff cannot run with scipy version less '+\
           'than 0.15.0, and requires the PIL installed; scipy version is '+\
           str(scipy.__version__)
         self.__same = False
-        return(self.__same, self.__messages)
+        return(self.__same, self.self.__message)
       try:
         # RAK - The original line...
         # test_image = imread(open(test_filename,'r'))
         # ...didn't work on Windows Python because it couldn't sense the file type
         test_image = imread(test_filename)
       except IOError:
-        self.__messages += 'Unrecognized file type for test image in scipy.imread: '+test_filename
+        self.self.__message += 'Unrecognized file type for test image in scipy.imread: '+test_filename
         files_read = False
-        return (False, self.__messages)
+        return (False, self.self.__message)
       try:
         # RAK - The original line...
         # gold_image = imread(open(gold_filename,'r'))
         # ...didn't work on Windows Python because it couldn't sense the file type
         gold_image = imread(gold_filename)
       except IOError:
         files_read = False
-        self.__messages += 'Unrecognized file type for test image in scipy.imread: '+gold_filename
-        return (False, self.__messages)
+        self.self.__message += 'Unrecognized file type for test image in scipy.imread: '+gold_filename
+        return (False, self.self.__message)
       #first check dimensionality
       if gold_image.shape != test_image.shape:
-        self.__messages += 'Gold and test image are not the same shape: '+\
+        self.self.__message += 'Gold and test image are not the same shape: '+\
           str(gold_image.shape)+', '+str(test_image.shape)
         self.__same = False
-        return (self.__same, self.__messages)
-      #set default options
-      DU.set_default_options(self.__options)
+        return (self.__same, self.self.__message)
       #pixelwise comparison
       #TODO in the future we can add greyscale, normalized coloring, etc.
       # For now just do raw comparison of right/wrong pixels
       diff = gold_image - test_image
-      only_diffs = diff[abs(diff) > self.__options['zero_threshold']]
+      only_diffs = diff[abs(diff) > self.__zero_threshold]
       pct_num_diff = only_diffs.size/float(diff.size)
-      if pct_num_diff > self.__options['rel_err']:
-        self.__messages += 'Difference between images is too large:'+\
+      if pct_num_diff > self.__rel_err:
+        self.self.__message += 'Difference between images is too large:'+\
           ' %2.2f pct (allowable: %2.2f)' %(100*pct_num_diff,
-                                            100*self.__options['rel_err'])
+                                            100*self.__rel_err)
         self.__same = False
-    return (self.__same, self.__messages)
+    return (self.__same, self.__message)
+
+
+class ImageDiffer(Differ):
+  """
+  This is the class to use for handling the parameters block.
+  """
+
+  @staticmethod
+  def get_valid_params():
+    """
+      Returns the valid parameters for this class.
+      @ In, None
+      @ Out, params, _ValidParameters, return the parameters.
+    """
+    params = Differ.get_valid_params()
+    params.add_param('rel_err', '', 'Relative Error for image files')
+    params.add_param('zero_threshold', sys.float_info.min*4.0,
+                     'it represents the value below which a float is '+
+                     'considered zero (XML comparison only)')
+    return params
+
+  def __init__(self, name, params, test_dir):
+    """
+      Initializer for the class. Takes a String name and a dictionary params
+      @ In, name, string, name of the test.
+      @ In, params, dictionary, parameters for the class
+      @ In, test_dir, string, path to the test.
+      @ Out, None.
+    """
+    Differ.__init__(self, name, params, test_dir)
+    self.__zero_threshold = self.specs['zero_threshold']
+    if len(self.specs['rel_err']) > 0:
+      self.__rel_err = float(self.specs['rel_err'])
+    else:
+      self.__rel_err = 1e-10
+
+  def check_output(self):
+    """
+      Checks that the output matches the gold.
+      returns (same, message) where same is true if the
+      test passes, or false if the test failes.  message should
+      gives a human readable explaination of the differences.
+      @ In, None
+      @ Out, (same, message), same is true if the tests passes.
+    """
+    image_files = self._get_test_files()
+    gold_files = self._get_gold_files()
+    image_diff = ImageDiff(image_files,
+                                  gold_files,
+                                  relative_error=self.__rel_err,
+                                  zero_threshold=self.__zero_threshold)
+    return image_diff.diff()
+

scripts/TestHarness/testers/RavenFramework.py

@@ -25,7 +25,7 @@
 import XMLDiff
 import TextDiff
 import ExistsDiff
-from RAVENImageDiff import ImageDiff
+import RAVENImageDiff
 import RavenUtils
 
 # Set this outside the class because the framework directory is constant for
@@ -138,6 +138,7 @@ def __init__(self, name, params):
     self.__make_differ('xml', XMLDiff.XML, {"unordered":False})
     self.__make_differ('UnorderedXml', XMLDiff.XML, {"unordered":True})
     self.__make_differ('text', TextDiff.Text)
+    self.__make_differ('image', RAVENImageDiff.ImageDiff)
     self.required_executable = self.specs['required_executable']
     self.required_libraries = self.specs['required_libraries'].split(' ')\
       if len(self.specs['required_libraries']) > 0 else []
@@ -268,17 +269,4 @@ def process_results(self, _):
       @ In, ignored, string, output of test.
       @ Out, None
     """
-
-    #image
-    image_opts = {}
-    if 'rel_err'        in self.specs.keys():
-      image_opts['rel_err'] = self.specs['rel_err']
-    if 'zero_threshold' in self.specs.keys():
-      image_opts['zero_threshold'] = self.specs['zero_threshold']
-    img_diff = ImageDiff(self.specs['test_dir'], self.img_files, **image_opts)
-    (img_same, img_messages) = img_diff.diff()
-    if not img_same:
-      self.set_diff(img_messages)
-      return
-
     self.set_success()

scripts/TestHarness/testers/UnorderedCSVDiffer.py

@@ -42,13 +42,12 @@ def __init__(self, out_files, gold_files, relative_error=1e-10,
                absolute_check=False, zero_threshold=None, ignore_sign=False):
     """
       Create an UnorderedCSVDiffer class
-      Note naming conventions are out of our control due to MOOSE test harness standards.
-      @ In, test_dir, the directory where the test takes place
       @ In, out_files, the files to be compared.  They will be in test_dir + out_files
       @ In, gold_files, the files to be compared to the out_files.
       @ In, relative_error, float, optional, relative error
       @ In, absolute_check, bool, optional, if True then check absolute
          differences in the values instead of relative differences
+      @ In, zero_threshold, float, optional, if a number <= abs(zero_threshold) it will be considered 0
       @ In, ignore_sign, bool, optional, if True then the sign will be ignored during the comparison
       @ Out, None.
     """

tests/framework/user_guide/ForwardSamplingStrategies/tests

@@ -33,11 +33,15 @@
     type = 'RavenFramework'
     input = 'forwardSamplingSparseGrid.xml'
     UnorderedXml = 'RunDir/SparseGrid/rom_output.xml'
-    image = 'RunDir/SparseGrid/1-historyPlot_scatter-scatter.png RunDir/SparseGrid/1-samplesModelPlot3D_scatter-scatter.png RunDir/SparseGrid/1-samplesROMPlot3D_scatter-scatter.png'
     rel_err = 0.01
     required_libraries = 'PIL'
     zero_threshold = 1e-8
     max_time = 500
+  [./image_diff]
+   type = ImageDiffer
+   output = 'RunDir/SparseGrid/1-historyPlot_scatter-scatter.png RunDir/SparseGrid/1-samplesModelPlot3D_scatter-scatter.png RunDir/SparseGrid/1-samplesROMPlot3D_scatter-scatter.png'
+   rel_err = 0.1
+  [../]
   [../]
 
 []