Add failureDir

plugin/src/main/kotlin/com/facebook/testing/screenshot/build/PullScreenshotsTask.kt

@@ -80,6 +80,11 @@ open class PullScreenshotsTask : ScreenshotTask() {
           add(extension.recordDir)
         }
 
+        if (verify && extension.failureDir != null) {
+            add("--failure-dir")
+            add("${extension.failureDir}")
+        }
+
         if (extension.multipleDevices) {
           add("--multiple-devices")
           add("${extension.multipleDevices}")

plugin/src/main/kotlin/com/facebook/testing/screenshot/build/ScreenshotsPlugin.kt

@@ -34,6 +34,8 @@ open class ScreenshotsPluginExtension {
   var pythonExecutable = "python"
   /** The directory to compare screenshots from in verify only mode */
   var referenceDir: String? = null
+  /** The directory to save failed screenshots */
+  var failureDir: String? = null
 }
 
 class ScreenshotsPlugin : Plugin<Project> {

plugin/src/py/android_screenshot_tests/pull_screenshots.py

@@ -486,7 +486,8 @@ def pull_screenshots(process,
                      verify=None,
                      opt_generate_png=None,
                      test_img_api=None,
-                     old_imgs_data=None):
+                     old_imgs_data=None,
+                     failure_dir=None):
     if not perform_pull and temp_dir is None:
         raise RuntimeError("""You must supply a directory for temp_dir if --no-pull is present""")
 
@@ -508,10 +509,19 @@ def pull_screenshots(process,
     record_dir = join(record, device_name) if record and device_name else record
     verify_dir = join(verify, device_name) if verify and device_name else verify
 
+    if failure_dir:
+        failure_dir = join(failure_dir, device_name) if device_name else failure_dir
+        if not os.path.exists(failure_dir):
+            os.makedirs(failure_dir)
+
+    print('RecordDir: %s' % record_dir)
+    print('VerifyDir: %s' % verify_dir)
+    # print('FailureDir: %s' % failure_dir)
+
     if record or verify:
         # don't import this early, since we need PIL to import this
         from .recorder import Recorder
-        recorder = Recorder(temp_dir, record_dir or verify_dir)
+        recorder = Recorder(temp_dir, record_dir or verify_dir, failure_dir)
         if verify:
             recorder.verify()
         else:
@@ -539,7 +549,7 @@ def main(argv):
         opt_list, rest_args = getopt.gnu_getopt(
             argv[1:],
             "eds:",
-            ["generate-png=", "filter-name-regex=", "apk", "record=", "verify=", "temp-dir=",
+            ["generate-png=", "filter-name-regex=", "apk", "record=", "verify=", "failure-dir=", "temp-dir=",
              "no-pull", "multiple-devices="])
     except getopt.GetoptError:
         usage()
@@ -590,7 +600,8 @@ def main(argv):
                          record=opts.get('--record'),
                          verify=opts.get('--verify'),
                          adb_puller=SimplePuller(puller_args),
-                         device_name_calculator=device_calculator)
+                         device_name_calculator=device_calculator,
+                         failure_dir=opts.get("--failure-dir"))
 
 
 if __name__ == '__main__':

plugin/src/py/android_screenshot_tests/recorder.py

@@ -23,7 +23,7 @@
 import sys
 
 from os.path import join
-from PIL import Image, ImageChops
+from PIL import Image, ImageChops, ImageDraw
 
 from . import common
 import shutil
@@ -33,10 +33,11 @@ class VerifyError(Exception):
     pass
 
 class Recorder:
-    def __init__(self, input, output):
+    def __init__(self, input, output, failure_output):
         self._input = input
         self._output = output
         self._realoutput = output
+        self._failure_output = failure_output
 
     def _get_image_size(self, file_name):
         with Image.open(file_name) as im:
@@ -87,11 +88,20 @@ def _clean(self):
             shutil.rmtree(self._output)
         os.makedirs(self._output)
 
-    def _is_image_same(self, file1, file2):
+    def _is_image_same(self, file1, file2, file3):
         with Image.open(file1) as im1, Image.open(file2) as im2:
             diff_image = ImageChops.difference(im1, im2)
             try:
-                return diff_image.getbbox() is None
+                diff = diff_image.getbbox()
+                if diff is None:
+                    return True
+                else:
+                    if file3:
+                        diff_list = list(diff) if diff else []
+                        draw = ImageDraw.Draw(im2)
+                        draw.rectangle(diff_list, outline = (255,0,0))
+                        im2.save(file3)
+                    return False
             finally:
                 diff_image.close()
 
@@ -104,11 +114,31 @@ def verify(self):
         self._record()
 
         root = self._get_metadata_root()
+        failures = []
         for screenshot in root.iter("screenshot"):
             name = screenshot.find('name').text + ".png"
             actual = join(self._output, name)
             expected = join(self._realoutput, name)
-            if not self._is_image_same(expected, actual):
-                raise VerifyError("Image %s is not same as %s" % (actual, expected))
+            if self._failure_output:
+                diff_name = screenshot.find('name').text + "_diff.png"
+                diff = join(self._failure_output, diff_name)
+
+                if not self._is_image_same(expected, actual, diff):
+                    expected_name = screenshot.find('name').text + "_expected.png"
+                    actual_name = screenshot.find('name').text + "_actual.png"
+
+                    shutil.copy(actual, join(self._failure_output, actual_name))
+                    shutil.copy(expected, join(self._failure_output, expected_name))
+
+                    failures.append((expected, actual))
+            else:
+                if not self._is_image_same(expected, actual, None):
+                    raise VerifyError("Image %s is not same as %s" % (expected, actual))                  
+
+        if failures:
+            reason = ''
+            for expected, actual in failures:
+                reason = reason + "\nImage %s is not same as %s" % (expected, actual)
+            raise VerifyError(reason)
 
         shutil.rmtree(self._output)

plugin/src/py/android_screenshot_tests/test_recorder.py

@@ -26,8 +26,9 @@ class TestRecorder(unittest.TestCase):
     def setUp(self):
         self.outputdir = tempfile.mkdtemp()
         self.inputdir = tempfile.mkdtemp()
+        self.failureDir = tempfile.mkdtemp()
         self.tmpimages = []
-        self.recorder = Recorder(self.inputdir, self.outputdir)
+        self.recorder = Recorder(self.inputdir, self.outputdir, self.failureDir)
 
     def create_temp_image(self, name, dimens, color):
         im = Image.new("RGBA", dimens, color)
@@ -188,13 +189,29 @@ def test_verify_failure(self):
 
         self.recorder.record()
         os.unlink(join(self.inputdir, "foobar.png"))
-        self.create_temp_image("foobar.png", (10, 10), "red")
+        self.create_temp_image("foobar.png", (11, 11), "green")
 
         try:
             self.recorder.verify()
             self.fail("expected exception")
         except VerifyError:
             pass  # expected
 
+        self.assertTrue(os.path.exists(join(self.failureDir, "foobar_actual.png")))
+        self.assertTrue(os.path.exists(join(self.failureDir, "foobar_expected.png")))
+        self.assertTrue(os.path.exists(join(self.failureDir, "foobar_diff.png")))
+
+        # check colored diff
+        with Image.open(join(self.failureDir, "foobar_diff.png")) as im:
+            (w, h) = im.size
+            self.assertEqual(11, w)
+            self.assertEqual(11, h)
+
+            self.assertEqual((255, 0, 0, 255), im.getpixel((0, 1)))
+            self.assertEqual((255, 0, 0, 255), im.getpixel((10, 1)))
+
+            self.assertEqual((0, 128, 0, 255), im.getpixel((1, 1)))
+            self.assertEqual((0, 128, 0, 255), im.getpixel((9, 1)))
+
 if __name__ == '__main__':
     unittest.main()