Add verbose and optimization args for parity tests (Gelu, Layernorm, …

onnxruntime/python/tools/transformers/optimizer.py

@@ -60,6 +60,7 @@ def optimize_by_onnxruntime(
  optimized_model_path: Optional[str] = None,
  opt_level: Optional[int] = 99,
  disabled_optimizers=[],
+ verbose=False,
 ) -> str:
  """
  Use onnxruntime to optimize model.
@@ -98,6 +99,10 @@ def optimize_by_onnxruntime(
 
  sess_options.optimized_model_filepath = optimized_model_path
 
+ if verbose:
+ print("Using onnxruntime to optimize model - Debug level Set to verbose")
+ sess_options.log_severity_level = 0
+
  kwargs = {}
  if disabled_optimizers:
  kwargs["disabled_optimizers"] = disabled_optimizers
@@ -114,7 +119,6 @@ def optimize_by_onnxruntime(
  elif torch_version.hip:
  gpu_ep.append("MIGraphXExecutionProvider")
  gpu_ep.append("ROCMExecutionProvider")
-
  session = onnxruntime.InferenceSession(onnx_model_path, sess_options, providers=gpu_ep, **kwargs)
  assert not set(onnxruntime.get_available_providers()).isdisjoint(
  ["CUDAExecutionProvider", "ROCMExecutionProvider", "MIGraphXExecutionProvider"]
@@ -189,6 +193,7 @@ def optimize_model(
  opt_level: Optional[int] = None,
  use_gpu: bool = False,
  only_onnxruntime: bool = False,
+ verbose=False,
 ):
  """Optimize Model by OnnxRuntime and/or python fusion logic.
 
@@ -260,6 +265,7 @@ def optimize_model(
  use_gpu=use_gpu,
  opt_level=opt_level,
  disabled_optimizers=disabled_optimizers,
+ verbose=verbose,
  )
  elif opt_level == 1:
  # basic optimizations (like constant folding and cast elimination) are not specified to execution provider.
@@ -269,6 +275,7 @@ def optimize_model(
  use_gpu=False,
  opt_level=1,
  disabled_optimizers=disabled_optimizers,
+ verbose=verbose,
  )
 
  if only_onnxruntime and not temp_model_path:

onnxruntime/test/python/transformers/parity_utilities.py

@@ -3,14 +3,44 @@
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
 # -------------------------------------------------------------------------
-
+import argparse
 import os
 import sys
 
 import numpy
 import torch
 
 
+def parse_arguments(namespace_filter=None):
+
+ parser = argparse.ArgumentParser()
+
+ # useful EPs that don't require the use of optmizer.py
+ parser.add_argument(
+ "-n",
+ "--no_optimize",
+ required=False,
+ action="store_false",
+ default=True,
+ dest="optimize",
+ help="Turn off onnxruntime optimizers (Default off optimizers ON)",
+ )
+
+ # useful for debugging and viewing state during test runs
+ parser.add_argument(
+ "-l",
+ "--log_verbose",
+ required=False,
+ action="store_true",
+ default=False,
+ help="Set Onnxruntime log_serverity_level=0 (VERBOSE) ",
+ )
+
+ args, remaining_args = parser.parse_known_args(namespace=namespace_filter)
+
+ return args, sys.argv[:1] + remaining_args
+
+
 def find_transformers_source(sub_dir_paths=[]):
  source_dir = os.path.join(
  os.path.dirname(__file__),
@@ -74,13 +104,16 @@ def optimize_onnx(
  expected_op=None,
  use_gpu=False,
  opt_level=None,
+ verbose=False,
 ):
  if find_transformers_source():
  from optimizer import optimize_model
  else:
  from onnxruntime.transformers.optimizer import optimize_model
 
- onnx_model = optimize_model(input_onnx_path, model_type="gpt2", use_gpu=use_gpu, opt_level=opt_level)
+ onnx_model = optimize_model(
+ input_onnx_path, model_type="gpt2", use_gpu=use_gpu, opt_level=opt_level, verbose=verbose
+ )
  onnx_model.save_model_to_file(optimized_onnx_path)
 
  if expected_op is not None:
@@ -130,21 +163,26 @@ def compare_outputs(torch_outputs, ort_outputs, atol=1e-06, verbose=True):
  return is_all_close, max(max_abs_diff)
 
 
-def create_ort_session(onnx_model_path, use_gpu=True):
+def create_ort_session(onnx_model_path, use_gpu=True, optimized=True, verbose=False):
  from onnxruntime import GraphOptimizationLevel, InferenceSession, SessionOptions
- from onnxruntime import __version__ as onnxruntime_version
 
  sess_options = SessionOptions()
  sess_options.graph_optimization_level = GraphOptimizationLevel.ORT_DISABLE_ALL
  sess_options.intra_op_num_threads = 2
  sess_options.log_severity_level = 2
+
+ if verbose:
+ sess_options.log_severity_level = 0
+
  execution_providers = ["CPUExecutionProvider"]
 
  if use_gpu:
  if torch.version.cuda:
  execution_providers.append("CUDAExecutionProvider")
  elif torch.version.hip:
- execution_providers.append("MIGraphXExecutionProvider")
+ if not optimized:
+ execution_providers.append("MIGraphXExecutionProvider")
+
  execution_providers.append("ROCMExecutionProvider")
 
  return InferenceSession(onnx_model_path, sess_options, providers=execution_providers)
@@ -172,7 +210,7 @@ def run_parity(
  passed_cases = 0
  max_diffs = []
  printed = False # print only one sample
- ort_session = create_ort_session(onnx_model_path, device.type == "cuda")
+ ort_session = create_ort_session(onnx_model_path, device.type == "cuda", optimized=optimized, verbose=verbose)
  for i in range(test_cases):
  input_hidden_states = create_inputs(batch_size, sequence_length, hidden_size, float16, device)
 

onnxruntime/test/python/transformers/test_parity_gelu.py

@@ -85,6 +85,7 @@ def run(
  formula=0,
  sequence_length=2,
  fp32_gelu_op=True,
+ verbose=False,
 ):
  test_name = f"device={device}, float16={float16}, optimized={optimized}, batch_size={batch_size}, sequence_length={sequence_length}, hidden_size={hidden_size}, formula={formula}, fp32_gelu_op={fp32_gelu_op}"
  print(f"\nTesting: {test_name}")
@@ -108,6 +109,7 @@ def run(
  Gelu.get_fused_op(formula),
  use_gpu=use_gpu,
  opt_level=2 if use_gpu else None,
+ verbose=verbose,
  )
  onnx_path = optimized_onnx_path
  else:
@@ -123,7 +125,7 @@ def run(
  device,
  optimized,
  test_cases,
- verbose=False,
+ verbose,
  )
 
  # clean up onnx file
@@ -135,8 +137,10 @@ def run(
 
 
 class TestGeluParity(unittest.TestCase):
+ verbose = False
+ optimized = True
+
  def setUp(self):
- self.optimized = True # Change it to False if you want to test parity of non optimized ONNX
  self.test_cases = 100 # Number of test cases per test run
  self.sequence_length = 2
  self.hidden_size = 768
@@ -159,6 +163,7 @@ def run_test(
  formula,
  enable_assert=True,
  fp32_gelu_op=True,
+ verbose=False,
  ):
  if float16 and device.type == "cpu": # CPU does not support FP16
  return
@@ -172,11 +177,12 @@ def run_test(
  formula,
  self.sequence_length,
  fp32_gelu_op,
+ verbose,
  )
  if enable_assert:
  self.assertTrue(num_failure == 0, "Failed: " + test_name)
 
- def run_one(self, optimized, device, hidden_size=768, formula=0):
+ def run_one(self, optimized, device, hidden_size=768, formula=0, verbose=False):
  for batch_size in [4]:
  self.run_test(
  batch_size,
@@ -186,6 +192,7 @@ def run_one(self, optimized, device, hidden_size=768, formula=0):
  device=device,
  formula=formula,
  enable_assert=formula in self.formula_must_pass,
+ verbose=verbose,
  )
 
  self.run_test(
@@ -197,6 +204,7 @@ def run_one(self, optimized, device, hidden_size=768, formula=0):
  formula=formula,
  enable_assert=formula in self.formula_must_pass,
  fp32_gelu_op=True,
+ verbose=verbose,
  )
 
  self.run_test(
@@ -208,12 +216,13 @@ def run_one(self, optimized, device, hidden_size=768, formula=0):
  formula=formula,
  enable_assert=formula in self.formula_must_pass,
  fp32_gelu_op=False,
+ verbose=verbose,
  )
 
  def test_cpu(self):
  cpu = torch.device("cpu")
  for i in self.formula_to_test:
- self.run_one(self.optimized, cpu, hidden_size=self.hidden_size, formula=i)
+ self.run_one(self.optimized, cpu, hidden_size=self.hidden_size, formula=i, verbose=self.verbose)
 
  def test_cuda(self):
  if not torch.cuda.is_available():
@@ -223,8 +232,13 @@ def test_cuda(self):
  else:
  gpu = torch.device("cuda")
  for i in self.formula_to_test:
- self.run_one(self.optimized, gpu, hidden_size=self.hidden_size, formula=i)
+ self.run_one(self.optimized, gpu, hidden_size=self.hidden_size, formula=i, verbose=self.verbose)
 
 
 if __name__ == "__main__":
- unittest.main()
+ args, remaining_args = parse_arguments(namespace_filter=unittest)
+
+ TestGeluParity.verbose = args.log_verbose
+ TestGeluParity.optimized = args.optimize
+
+ unittest.main(argv=remaining_args)