fix: Upgrade perf benchmark system

tools/perf/benchmark.sh

@@ -14,7 +14,7 @@ echo "Benchmarking VGG16 model"
 for bs in ${batch_sizes[@]}
 do
   python perf_run.py --model ${MODELS_DIR}/vgg16_scripted.jit.pt \
-                     --model_torch ${MODELS_DIR}/vgg16_pytorch.pt \
+                     --model_torch vgg16 \
                      --precision fp32,fp16 --inputs="(${bs}, 3, 224, 224)" \
                      --batch_size ${bs} \
                      --truncate \
@@ -27,7 +27,7 @@ echo "Benchmarking AlexNet model"
 for bs in ${batch_sizes[@]}
 do
   python perf_run.py --model ${MODELS_DIR}/alexnet_scripted.jit.pt \
-                     --model_torch ${MODELS_DIR}/alexnet_pytorch.pt \
+                     --model_torch alexnet \
                      --precision fp32,fp16 --inputs="(${bs}, 3, 227, 227)" \
                      --batch_size ${bs} \
                      --truncate \
@@ -40,7 +40,7 @@ echo "Benchmarking Resnet50 model"
 for bs in ${batch_sizes[@]}
 do
   python perf_run.py --model ${MODELS_DIR}/resnet50_scripted.jit.pt \
-                     --model_torch ${MODELS_DIR}/resnet50_pytorch.pt \
+                     --model_torch resnet50 \
                      --precision fp32,fp16 --inputs="(${bs}, 3, 224, 224)" \
                      --batch_size ${bs} \
                      --truncate \
@@ -53,7 +53,7 @@ echo "Benchmarking VIT model"
 for bs in ${batch_sizes[@]}
 do
   python perf_run.py --model ${MODELS_DIR}/vit_scripted.jit.pt \
-                     --model_torch ${MODELS_DIR}/vit_pytorch.pt \
+                     --model_torch vit \
                      --precision fp32,fp16 --inputs="(${bs}, 3, 224, 224)" \
                      --batch_size ${bs} \
                      --truncate \
@@ -66,19 +66,20 @@ echo "Benchmarking VIT Large model"
 for bs in ${large_model_batch_sizes[@]}
 do
   python perf_run.py --model ${MODELS_DIR}/vit_large_scripted.jit.pt \
-                     --model_torch ${MODELS_DIR}/vit_large_pytorch.pt \
+                     --model_torch vit_large \
                      --precision fp32,fp16 --inputs="(${bs}, 3, 224, 224)" \
                      --truncate \
                      --batch_size ${bs} \
                      --backends torch,ts_trt,dynamo,torch_compile,inductor \
                      --report "vit_large_perf_bs${bs}.txt"
+done
 
 # Benchmark EfficientNet-B0 model
 echo "Benchmarking EfficientNet-B0 model"
 for bs in ${batch_sizes[@]}
 do
   python perf_run.py --model ${MODELS_DIR}/efficientnet_b0_scripted.jit.pt \
-                     --model_torch ${MODELS_DIR}/efficientnet_b0_pytorch.pt \
+                     --model_torch efficientnet_b0 \
                      --precision fp32,fp16 --inputs="(${bs}, 3, 224, 224)" \
                      --batch_size ${bs} \
                      --truncate \
@@ -90,7 +91,7 @@ done
 echo "Benchmarking SD UNet model"
 for bs in ${large_model_batch_sizes[@]}
 do
-  python perf_run.py --model_torch ${MODELS_DIR}/sd_unet_pytorch.pt \
+  python perf_run.py --model_torch sd_unet \
                      --precision fp32,fp16 --inputs="(${bs}, 4, 128, 128)@fp16;(${bs})@fp16;(${bs}, 1, 768)@fp16" \
                      --batch_size ${bs} \
                      --backends torch,dynamo,torch_compile,inductor \

tools/perf/hub.py

@@ -67,9 +67,25 @@ def download_models(version_matches, manifest):
     # Download all models if torch version is different than model version
     if not version_matches:
         for n, m in BENCHMARK_MODELS.items():
+            # Ensure all specified desired model formats exist and are valid
+            assert all(
+                (path in VALID_PATHS)
+                for path in (
+                    m["path"] if isinstance(m["path"], (list, tuple)) else [m["path"]]
+                )
+            ), "Not all 'path' attributes in BENCHMARK_MODELS are valid"
+
             manifest = get(n, m, manifest)
     else:
         for n, m in BENCHMARK_MODELS.items():
+            # Ensure all specified desired model formats exist and are valid
+            assert all(
+                (path in VALID_PATHS)
+                for path in (
+                    m["path"] if isinstance(m["path"], (list, tuple)) else [m["path"]]
+                )
+            ), "Not all 'path' attributes in BENCHMARK_MODELS are valid"
+
             scripted_filename = "models/" + n + "_scripted.jit.pt"
             traced_filename = "models/" + n + "_traced.jit.pt"
             pytorch_filename = "models/" + n + "_pytorch.pt"
@@ -147,12 +163,4 @@ def main():
 
 
 if __name__ == "__main__":
-    # Ensure all specified desired model formats exist and are valid
-    paths = [
-        [m["path"]] if isinstance(m["path"], str) else m["path"]
-        for m in BENCHMARK_MODELS.values()
-    ]
-    assert all(
-        (path in VALID_PATHS) for path_list in paths for path in path_list
-    ), "Not all 'path' attributes in BENCHMARK_MODELS are valid"
     main()

tools/perf/perf_run.py

@@ -15,7 +15,6 @@
 # Importing supported Backends
 import torch
 import torch.backends.cudnn as cudnn
-import torch_tensorrt as torchtrt
 from utils import (
     BENCHMARK_MODELS,
     parse_backends,
@@ -24,6 +23,8 @@
     precision_to_dtype,
 )
 
+import torch_tensorrt as torchtrt
+
 WARMUP_ITER = 10
 results = []
 
@@ -465,6 +466,7 @@ def recordStats(backend, timings, precision, batch_size=1, compile_time_s=None):
         "Mean(FPS)": speed_mean,
         "Median-Latency(ms)": time_med * 1000,
         "Mean-Latency(ms)": time_mean * 1000,
+        "Latency-StdDev(ms)": time_std * 1000,
         "Compile Time(s)": compile_time_s,
     }
     results.append(stats)
@@ -550,16 +552,6 @@ def recordStats(backend, timings, precision, batch_size=1, compile_time_s=None):
         model_torch = torch.load(model_name_torch).eval().cuda()
     elif model_name_torch in BENCHMARK_MODELS:
         model_torch = BENCHMARK_MODELS[model_name_torch]["model"].eval().cuda()
-        if model_name_torch == "bert_base_uncased":
-            from transformers.utils.fx import symbolic_trace as transformers_trace
-
-            model_torch = (
-                transformers_trace(
-                    model_torch, input_names=["input_ids", "attention_mask"]
-                )
-                .eval()
-                .cuda()
-            )
 
     # If neither model type was provided
     if (model is None) and (model_torch is None):

tools/perf/utils.py

@@ -1,46 +1,83 @@
+from typing import Optional, Sequence, Union
+
 import custom_models as cm
 import timm
 import torch
-import torch_tensorrt
 import torchvision.models as models
 
-BENCHMARK_MODELS = {
-    "vgg16": {
-        "model": models.vgg16(weights=models.VGG16_Weights.DEFAULT),
-        "path": ["script", "pytorch"],
-    },
-    "alexnet": {
-        "model": models.alexnet(weights=models.AlexNet_Weights.DEFAULT),
-        "path": ["script", "pytorch"],
-    },
-    "resnet50": {
-        "model": models.resnet50(weights=None),
-        "path": ["script", "pytorch"],
-    },
-    "efficientnet_b0": {
-        "model": timm.create_model("efficientnet_b0", pretrained=True),
-        "path": ["script", "pytorch"],
-    },
-    "vit": {
-        "model": timm.create_model("vit_base_patch16_224", pretrained=True),
-        "path": ["script", "pytorch"],
-    },
-    "vit_large": {
-        "model": timm.create_model("vit_giant_patch14_224", pretrained=False),
-        "path": ["script", "pytorch"],
-    },
-    "bert_base_uncased": {
-        "model": cm.BertModule(),
-        "inputs": cm.BertInputs(),
-        "path": ["trace", "pytorch"],
-    },
-    "sd_unet": {
-        "model": cm.StableDiffusionUnet(),
-        "path": "pytorch",
-    },
+import torch_tensorrt
+
+BENCHMARK_MODEL_NAMES = {
+    "vgg16",
+    "alexnet",
+    "resnet50",
+    "efficientnet_b0",
+    "vit",
+    "vit_large",
+    "bert_base_uncased",
+    "sd_unet",
 }
 
 
+class ModelStorage:
+    def __contains__(self, name: str):
+        return name in BENCHMARK_MODEL_NAMES
+
+    def __getitem__(self, name: str):
+        assert name in BENCHMARK_MODEL_NAMES
+
+        if name == "vgg16":
+            return {
+                "model": models.vgg16(weights=models.VGG16_Weights.DEFAULT),
+                "path": ["script", "pytorch"],
+            }
+        elif name == "alexnet":
+            return {
+                "model": models.alexnet(weights=models.AlexNet_Weights.DEFAULT),
+                "path": ["script", "pytorch"],
+            }
+        elif name == "resnet50":
+            return {
+                "model": models.resnet50(weights=None),
+                "path": ["script", "pytorch"],
+            }
+        elif name == "efficientnet_b0":
+            return {
+                "model": timm.create_model("efficientnet_b0", pretrained=True),
+                "path": ["script", "pytorch"],
+            }
+        elif name == "vit":
+            return {
+                "model": timm.create_model("vit_base_patch16_224", pretrained=True),
+                "path": ["script", "pytorch"],
+            }
+        elif name == "vit_large":
+            return {
+                "model": timm.create_model("vit_giant_patch14_224", pretrained=False),
+                "path": ["script", "pytorch"],
+            }
+        elif name == "bert_base_uncased":
+            return {
+                "model": cm.BertModule(),
+                "inputs": cm.BertInputs(),
+                "path": ["trace", "pytorch"],
+            }
+        elif name == "sd_unet":
+            return {
+                "model": cm.StableDiffusionUnet(),
+                "path": "pytorch",
+            }
+        else:
+            raise AssertionError(f"Invalid model name {name}")
+
+    def items(self):
+        for name in BENCHMARK_MODEL_NAMES:
+            yield name, self.__getitem__(name)
+
+
+BENCHMARK_MODELS = ModelStorage()
+
+
 def precision_to_dtype(pr):
     if pr == "fp32":
         return torch.float