Add resnet56 short tests. (tensorflow#6101)

* Add resnet56 short tests.
- created base benchmark module
- renamed accuracy test class to contain the word Accuracy
which will result in a need to update all the jobs
and a loss of history but is worth it.
- short tests are mostly copied from shining with oss refactor

* Address feedback.

* Move flag_methods to init
- Address setting default flags repeatedly.

* Rename accuracy tests.

* Lint errors resolved.

* fix model_dir set to flags.data_dir.

* fixed not fulling pulling out flag_methods.

official/resnet/keras/keras_benchmark.py

@@ -0,0 +1,96 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Executes Keras benchmarks and accuracy tests."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+
+from absl import flags
+from absl.testing import flagsaver
+import tensorflow as tf  # pylint: disable=g-bad-import-order
+
+FLAGS = flags.FLAGS
+
+
+class KerasBenchmark(object):
+  """Base benchmark class with methods to simplify testing."""
+  local_flags = None
+
+  def __init__(self, output_dir=None, default_flags=None, flag_methods=None):
+    self.oss_report_object = None
+    self.output_dir = output_dir
+    self.default_flags = default_flags or {}
+    self.flag_methods = flag_methods or {}
+
+  def _get_model_dir(self, folder_name):
+    return os.path.join(self.output_dir, folder_name)
+
+  def _setup(self):
+    """Sets up and resets flags before each test."""
+    tf.logging.set_verbosity(tf.logging.DEBUG)
+    if KerasBenchmark.local_flags is None:
+      for flag_method in self.flag_methods:
+        flag_method()
+      # Loads flags to get defaults to then override. List cannot be empty.
+      flags.FLAGS(['foo'])
+      # Overrides flag values with defaults for the class of tests.
+      for k, v in self.default_flags.items():
+        setattr(FLAGS, k, v)
+      saved_flag_values = flagsaver.save_flag_values()
+      KerasBenchmark.local_flags = saved_flag_values
+    else:
+      flagsaver.restore_flag_values(KerasBenchmark.local_flags)
+
+  def fill_report_object(self, stats, top_1_max=None, top_1_min=None,
+                         log_steps=None, total_batch_size=None, warmup=1):
+    """Fills report object to report results.
+
+    Args:
+      stats: dict returned from keras models with known entries.
+      top_1_max: highest passing level for top_1 accuracy.
+      top_1_min: lowest passing level for top_1 accuracy.
+      log_steps: How often the log was created for stats['step_timestamp_log'].
+      total_batch_size: Global batch-size.
+      warmup: number of entries in stats['step_timestamp_log'] to ignore.
+    """
+    if self.oss_report_object:
+
+      if 'accuracy_top_1' in stats:
+        self.oss_report_object.add_top_1(stats['accuracy_top_1'],
+                                         expected_min=top_1_min,
+                                         expected_max=top_1_max)
+        self.oss_report_object.add_other_quality(
+            stats['training_accuracy_top_1'],
+            'top_1_train_accuracy')
+      if (warmup and
+          'step_timestamp_log' in stats and
+          len(stats['step_timestamp_log']) > warmup):
+        # first entry in the time_log is start of step 1. The rest of the
+        # entries are the end of each step recorded
+        time_log = stats['step_timestamp_log']
+        elapsed = time_log[-1].timestamp - time_log[warmup].timestamp
+        num_examples = (total_batch_size * log_steps * (len(time_log)-warmup-1))
+        examples_per_sec = num_examples / elapsed
+        self.oss_report_object.add_examples_per_second(examples_per_sec)
+
+      if 'avg_exp_per_second' in stats:
+        self.oss_report_object.add_result(stats['avg_exp_per_second'],
+                                          'avg_exp_per_second',
+                                          'exp_per_second')
+    else:
+      raise ValueError('oss_report_object has not been set.')

official/resnet/keras/keras_cifar_benchmark.py

@@ -18,118 +18,203 @@
 from __future__ import division
 from __future__ import print_function
 
-import os
-
 from absl import flags
-from absl.testing import flagsaver
-import tensorflow as tf  # pylint: disable=g-bad-import-order
 
 from official.resnet import cifar10_main as cifar_main
+from official.resnet.keras import keras_benchmark
 from official.resnet.keras import keras_cifar_main
 from official.resnet.keras import keras_common
 
-
 DATA_DIR = '/data/cifar10_data/cifar-10-batches-bin'
 MIN_TOP_1_ACCURACY = 0.925
 MAX_TOP_1_ACCURACY = 0.938
 
+FLAGS = flags.FLAGS
 
-class KerasCifar10BenchmarkTests(object):
-  """Benchmarks and accuracy tests for KerasCifar10."""
 
-  local_flags = None
+class Resnet56KerasAccuracy(keras_benchmark.KerasBenchmark):
+  """Accuracy tests for ResNet56 Keras CIFAR-10."""
 
   def __init__(self, output_dir=None):
-    self.oss_report_object = None
-    self.output_dir = output_dir
+    flag_methods = [keras_common.define_keras_flags,
+                    cifar_main.define_cifar_flags]
+
+    super(Resnet56KerasAccuracy, self).__init__(output_dir=output_dir,
+                                                flag_methods=flag_methods)
 
-  def keras_resnet56_1_gpu(self):
+  def benchmark_graph_1_gpu(self):
     """Test keras based model with Keras fit and distribution strategies."""
     self._setup()
-    flags.FLAGS.num_gpus = 1
-    flags.FLAGS.data_dir = DATA_DIR
-    flags.FLAGS.batch_size = 128
-    flags.FLAGS.train_epochs = 182
-    flags.FLAGS.model_dir = self._get_model_dir('keras_resnet56_1_gpu')
-    flags.FLAGS.dtype = 'fp32'
-    stats = keras_cifar_main.run(flags.FLAGS)
-    self._fill_report_object(stats)
-
-  def keras_resnet56_eager_1_gpu(self):
+    FLAGS.num_gpus = 1
+    FLAGS.data_dir = DATA_DIR
+    FLAGS.batch_size = 128
+    FLAGS.train_epochs = 182
+    FLAGS.model_dir = self._get_model_dir('keras_resnet56_1_gpu')
+    FLAGS.dtype = 'fp32'
+    stats = keras_cifar_main.run(FLAGS)
+    self.fill_report_object(stats, FLAGS.batch_size)
+
+  def benchmark_1_gpu(self):
     """Test keras based model with eager and distribution strategies."""
     self._setup()
-    flags.FLAGS.num_gpus = 1
-    flags.FLAGS.data_dir = DATA_DIR
-    flags.FLAGS.batch_size = 128
-    flags.FLAGS.train_epochs = 182
-    flags.FLAGS.model_dir = self._get_model_dir('keras_resnet56_eager_1_gpu')
-    flags.FLAGS.dtype = 'fp32'
-    flags.FLAGS.enable_eager = True
+    FLAGS.num_gpus = 1
+    FLAGS.data_dir = DATA_DIR
+    FLAGS.batch_size = 128
+    FLAGS.train_epochs = 182
+    FLAGS.model_dir = self._get_model_dir('keras_resnet56_eager_1_gpu')
+    FLAGS.dtype = 'fp32'
+    FLAGS.enable_eager = True
     stats = keras_cifar_main.run(flags.FLAGS)
-    self._fill_report_object(stats)
+    self.fill_report_object(stats, FLAGS.batch_size)
 
-  def keras_resnet56_eager_2_gpu(self):
+  def benchmark_2_gpu(self):
     """Test keras based model with eager and distribution strategies."""
     self._setup()
-    flags.FLAGS.num_gpus = 2
-    flags.FLAGS.data_dir = DATA_DIR
-    flags.FLAGS.batch_size = 128
-    flags.FLAGS.train_epochs = 182
-    flags.FLAGS.model_dir = self._get_model_dir('keras_resnet56_eager_2_gpu')
-    flags.FLAGS.dtype = 'fp32'
-    flags.FLAGS.enable_eager = True
-    stats = keras_cifar_main.run(flags.FLAGS)
-    self._fill_report_object(stats)
-
-  def keras_resnet56_2_gpu(self):
+    FLAGS.num_gpus = 2
+    FLAGS.data_dir = DATA_DIR
+    FLAGS.batch_size = 128
+    FLAGS.train_epochs = 182
+    FLAGS.model_dir = self._get_model_dir('keras_resnet56_eager_2_gpu')
+    FLAGS.dtype = 'fp32'
+    FLAGS.enable_eager = True
+    stats = keras_cifar_main.run(FLAGS)
+    self.fill_report_object(stats, FLAGS.batch_size)
+
+  def benchmark_graph_2_gpu(self):
     """Test keras based model with Keras fit and distribution strategies."""
     self._setup()
-    flags.FLAGS.num_gpus = 2
-    flags.FLAGS.data_dir = DATA_DIR
-    flags.FLAGS.data_dir = self._get_model_dir('keras_resnet56_2_gpu')
-    flags.FLAGS.batch_size = 128
-    flags.FLAGS.train_epochs = 182
-    flags.FLAGS.model_dir = ''
-    flags.FLAGS.dtype = 'fp32'
-    stats = keras_cifar_main.run(flags.FLAGS)
-    self._fill_report_object(stats)
-
-  def keras_resnet56_no_dist_strat_1_gpu(self):
+    FLAGS.num_gpus = 2
+    FLAGS.data_dir = DATA_DIR
+    FLAGS.batch_size = 128
+    FLAGS.train_epochs = 182
+    FLAGS.model_dir = self._get_model_dir('keras_resnet56_2_gpu')
+    FLAGS.dtype = 'fp32'
+    stats = keras_cifar_main.run(FLAGS)
+    self.fill_report_object(stats, FLAGS.batch_size)
+
+  def benchmark_graph_1_gpu_no_dist_strat(self):
     """Test keras based model with Keras fit but not distribution strategies."""
     self._setup()
-    flags.FLAGS.turn_off_distribution_strategy = True
-    flags.FLAGS.num_gpus = 1
-    flags.FLAGS.data_dir = DATA_DIR
-    flags.FLAGS.batch_size = 128
-    flags.FLAGS.train_epochs = 182
-    flags.FLAGS.model_dir = self._get_model_dir(
+    FLAGS.turn_off_distribution_strategy = True
+    FLAGS.num_gpus = 1
+    FLAGS.data_dir = DATA_DIR
+    FLAGS.batch_size = 128
+    FLAGS.train_epochs = 182
+    FLAGS.model_dir = self._get_model_dir(
         'keras_resnet56_no_dist_strat_1_gpu')
-    flags.FLAGS.dtype = 'fp32'
-    stats = keras_cifar_main.run(flags.FLAGS)
-    self._fill_report_object(stats)
-
-  def _fill_report_object(self, stats):
-    if self.oss_report_object:
-      self.oss_report_object.add_top_1(stats['accuracy_top_1'],
-                                       expected_min=MIN_TOP_1_ACCURACY,
-                                       expected_max=MAX_TOP_1_ACCURACY)
-      self.oss_report_object.add_other_quality(stats['training_accuracy_top_1'],
-                                               'top_1_train_accuracy')
-    else:
-      raise ValueError('oss_report_object has not been set.')
-
-  def _get_model_dir(self, folder_name):
-    return os.path.join(self.output_dir, folder_name)
-
-  def _setup(self):
-    """Setups up and resets flags before each test."""
-    tf.logging.set_verbosity(tf.logging.DEBUG)
-    if KerasCifar10BenchmarkTests.local_flags is None:
-      keras_common.define_keras_flags()
-      cifar_main.define_cifar_flags()
-      # Loads flags to get defaults to then override. List cannot be empty.
-      flags.FLAGS(['foo'])
-      saved_flag_values = flagsaver.save_flag_values()
-      KerasCifar10BenchmarkTests.local_flags = saved_flag_values
-      return
-    flagsaver.restore_flag_values(KerasCifar10BenchmarkTests.local_flags)
+    FLAGS.dtype = 'fp32'
+    stats = keras_cifar_main.run(FLAGS)
+    self.fill_report_object(stats, FLAGS.batch_size)
+
+  def fill_report_object(self, stats, total_batch_size):
+    super(Resnet56KerasAccuracy, self).fill_report_object(
+        stats,
+        top_1_min=MIN_TOP_1_ACCURACY,
+        top_1_max=MAX_TOP_1_ACCURACY,
+        total_batch_size=total_batch_size,
+        log_steps=100)
+
+
+class Resnet56KerasBenchmarkBase(keras_benchmark.KerasBenchmark):
+  """Short performance tests for ResNet56 via Keras and CIFAR-10."""
+
+  def __init__(self, output_dir=None, default_flags=None):
+    flag_methods = [keras_common.define_keras_flags,
+                    cifar_main.define_cifar_flags]
+
+    super(Resnet56KerasBenchmarkBase, self).__init__(
+        output_dir=output_dir,
+        flag_methods=flag_methods,
+        default_flags=default_flags)
+
+  def _run_benchmark(self):
+    stats = keras_cifar_main.run(FLAGS)
+    self.fill_report_object(stats)
+
+  def benchmark_1_gpu_no_dist_strat(self):
+    self._setup()
+    FLAGS.num_gpus = 1
+    FLAGS.enable_eager = True
+    FLAGS.turn_off_distribution_strategy = True
+    FLAGS.batch_size = 128
+
+    self._run_benchmark()
+
+  def benchmark_graph_1_gpu_no_dist_strat(self):
+    self._setup()
+    FLAGS.num_gpus = 1
+    FLAGS.enable_eager = False
+    FLAGS.turn_off_distribution_strategy = True
+    FLAGS.batch_size = 128
+
+    self._run_benchmark()
+
+  def benchmark_1_gpu(self):
+    self._setup()
+    FLAGS.num_gpus = 1
+    FLAGS.enable_eager = True
+    FLAGS.turn_off_distribution_strategy = False
+    FLAGS.batch_size = 128
+
+    self._run_benchmark()
+
+  def benchmark_graph_1_gpu(self):
+    self._setup()
+    FLAGS.num_gpus = 1
+    FLAGS.enable_eager = False
+    FLAGS.turn_off_distribution_strategy = False
+    FLAGS.batch_size = 128
+
+    self._run_benchmark()
+
+  def benchmark_2_gpu(self):
+    self._setup()
+    FLAGS.num_gpus = 2
+    FLAGS.enable_eager = True
+    FLAGS.turn_off_distribution_strategy = False
+    FLAGS.batch_size = 128 * 2  # 2 GPUs
+
+    self._run_benchmark()
+
+  def benchmark_graph_2_gpu(self):
+    self._setup()
+    FLAGS.num_gpus = 2
+    FLAGS.enable_eager = False
+    FLAGS.turn_off_distribution_strategy = False
+    FLAGS.batch_size = 128 * 2  # 2 GPUs
+
+    self._run_benchmark()
+
+  def fill_report_object(self, stats):
+    super(Resnet56KerasBenchmarkBase, self).fill_report_object(
+        stats,
+        total_batch_size=FLAGS.batch_size,
+        log_steps=FLAGS.log_steps)
+
+
+class Resnet56KerasBenchmarkSynth(Resnet56KerasBenchmarkBase):
+  """Synthetic benchmarks for ResNet56 and Keras."""
+
+  def __init__(self, output_dir=None):
+    def_flags = {}
+    def_flags['skip_eval'] = True
+    def_flags['use_synthetic_data'] = True
+    def_flags['train_steps'] = 110
+    def_flags['log_steps'] = 10
+
+    super(Resnet56KerasBenchmarkSynth, self).__init__(output_dir=output_dir,
+                                                      default_flags=def_flags)
+
+
+class Resnet56KerasBenchmarkReal(Resnet56KerasBenchmarkBase):
+  """Real data benchmarks for ResNet56 and Keras."""
+
+  def __init__(self, output_dir=None):
+    def_flags = {}
+    def_flags['skip_eval'] = True
+    def_flags['data_dir'] = DATA_DIR
+    def_flags['train_steps'] = 110
+    def_flags['log_steps'] = 10
+
+    super(Resnet56KerasBenchmarkReal, self).__init__(output_dir=output_dir,
+                                                     default_flags=def_flags)