From 684f323b507126e2b47566e0b2a2090b2102bb83 Mon Sep 17 00:00:00 2001 From: Pablo Gonzalez Date: Mon, 27 Jun 2022 17:28:13 -0500 Subject: [PATCH 1/8] Update README (#1167) --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 7ac47f8254..36a0620bfc 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,20 @@ Please see the [MLPerf Inference benchmark paper](https://arxiv.org/abs/1911.025 primaryClass={cs.LG} } ``` +## MLPerf Inference v2.1 (submission 08/05/2022) +Use the r2.1 branch (```git checkout r2.1```) if you want to submit or reproduce v2.1 results. + +See the individual Readme files in the reference app for details. + +| model | reference app | framework | dataset | +| ---- | ---- | ---- | ---- | +| resnet50-v1.5 | [vision/classification_and_detection](https://github.com/mlcommons/inference/tree/master/vision/classification_and_detection) | tensorflow, pytorch, onnx | imagenet2012 | +| ssd-resnext50 800x800 | [vision/classification_and_detection](https://github.com/mlcommons/inference/tree/master/vision/classification_and_detection) | pytorch, onnx | openimages resized to 800x800| +| bert | [language/bert](https://github.com/mlcommons/inference/tree/master/language/bert) | tensorflow, pytorch, onnx | squad-1.1 | +| dlrm | [recommendation/dlrm](https://github.com/mlcommons/inference/tree/master/recommendation/dlrm/pytorch) | pytorch, tensorflow(?), onnx(?) | Criteo Terabyte | +| 3d-unet | [vision/medical_imaging/3d-unet-kits19](https://github.com/mlcommons/inference/tree/master/vision/medical_imaging/3d-unet-kits19) | pytorch, tensorflow, onnx | KiTS19 | +| rnnt | [speech_recognition/rnnt](https://github.com/mlcommons/inference/tree/master/speech_recognition/rnnt) | pytorch | OpenSLR LibriSpeech Corpus | + ## MLPerf Inference v2.0 (submission 02/25/2022) Use the r2.0 branch (```git checkout r2.0```) if you want to submit or reproduce v2.0 results. From b734af9019f69edd1fafeddfe674c5d3f8644d45 Mon Sep 17 00:00:00 2001 From: pgmpablo157321 Date: Tue, 28 Jun 2022 10:02:34 -0500 Subject: [PATCH 2/8] Change python to python3 --- .../tools/openimages_calibration_mlperf.sh | 4 ++-- .../classification_and_detection/tools/openimages_mlperf.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/vision/classification_and_detection/tools/openimages_calibration_mlperf.sh b/vision/classification_and_detection/tools/openimages_calibration_mlperf.sh index 002591a367..9744e065cd 100755 --- a/vision/classification_and_detection/tools/openimages_calibration_mlperf.sh +++ b/vision/classification_and_detection/tools/openimages_calibration_mlperf.sh @@ -50,7 +50,7 @@ MLPERF_CLASSES=('Airplane' 'Antelope' 'Apple' 'Backpack' 'Balloon' 'Banana' 'Whale' 'Wheel' 'Wheelchair' 'Whiteboard' 'Window' 'Wine' 'Wine glass' 'Woman' 'Zebra' 'Zucchini') -python openimages_calibration.py \ +python3 openimages_calibration.py \ --dataset-dir=${DATASET_PATH} \ --output-labels="openimages-mlperf.json" \ - --classes "${MLPERF_CLASSES[@]}" \ No newline at end of file + --classes "${MLPERF_CLASSES[@]}" diff --git a/vision/classification_and_detection/tools/openimages_mlperf.sh b/vision/classification_and_detection/tools/openimages_mlperf.sh index ca877f9fb6..00a331c8da 100755 --- a/vision/classification_and_detection/tools/openimages_mlperf.sh +++ b/vision/classification_and_detection/tools/openimages_mlperf.sh @@ -50,7 +50,7 @@ MLPERF_CLASSES=('Airplane' 'Antelope' 'Apple' 'Backpack' 'Balloon' 'Banana' 'Whale' 'Wheel' 'Wheelchair' 'Whiteboard' 'Window' 'Wine' 'Wine glass' 'Woman' 'Zebra' 'Zucchini') -python openimages.py \ +python3 openimages.py \ --dataset-dir=${DATASET_PATH} \ --output-labels="openimages-mlperf.json" \ - --classes "${MLPERF_CLASSES[@]}" \ No newline at end of file + --classes "${MLPERF_CLASSES[@]}" From 27adf9a5932fd549f97dc1713bfc4ab615a468c3 Mon Sep 17 00:00:00 2001 From: pgmpablo157321 Date: Fri, 24 Jun 2022 11:42:34 -0500 Subject: [PATCH 3/8] Update name and model scores --- tools/submission/submission-checker.py | 2 +- vision/classification_and_detection/README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/submission/submission-checker.py b/tools/submission/submission-checker.py index 6f0f1d7f37..b8ee91618f 100755 --- a/tools/submission/submission-checker.py +++ b/tools/submission/submission-checker.py @@ -610,7 +610,7 @@ "accuracy-target": { "resnet": ("acc", 76.46 * 0.99), # TODO: Update accuracy target for ssd-resnext50 - "ssd-resnext50": ("mAP", 37.5 * 0.99), + "ssd-resnext50": ("mAP", 37.55 * 0.99), "rnnt": ("WER", (100 - 7.452) * 0.99), "bert-99": ("F1", 90.874 * 0.99), "bert-99.9": ("F1", 90.874 * 0.999), diff --git a/vision/classification_and_detection/README.md b/vision/classification_and_detection/README.md index 2a3ffc98da..b7a1406cb1 100755 --- a/vision/classification_and_detection/README.md +++ b/vision/classification_and_detection/README.md @@ -28,8 +28,8 @@ You can find a short tutorial how to use this benchmark [here](https://github.co | ssd-resnet34 1200x1200 (removed since mlperf-v2.1)| pytorch | mAP 0.20 | coco resized to 1200x1200 | [from zenodo](https://zenodo.org/record/3236545/files/resnet34-ssd1200.pytorch) | [from mlperf](https://github.com/mlperf/inference/tree/master/others/cloud/single_stage_detector/pytorch) | fp32 | NCHW | | ssd-resnet34 1200x1200 (removed since mlperf-v2.1) | onnx | mAP 0.20 | coco resized to 1200x1200 | from zenodo [opset-8](https://zenodo.org/record/3228411/files/resnet34-ssd1200.onnx) | [from mlperf](https://github.com/mlperf/inference/tree/master/others/cloud/single_stage_detector) converted using the these [instructions](https://github.com/BowenBao/inference/tree/master/cloud/single_stage_detector/pytorch#6-onnx) | fp32 | Converted from pytorch model. | | ssd-resnet34 1200x1200 (removed since mlperf-v2.1) | onnx | mAP 0.20 | coco resized to 1200x1200 | from zenodo [opset-11](https://zenodo.org/record/4735664/files/ssd_resnet34_mAP_20.2.onnx) | [from zenodo](https://zenodo.org/record/3345892/files/tf_ssd_resnet34_22.1.zip) converted using [this script](https://github.com/mlcommons/inference/blob/master/vision/classification_and_detection/tools/convert-to-onnx.sh) | fp32 | Converted from the tensorflow model and uses the same interface as the tensorflow model. | -| retinanet-resnext50 800x800 | pytorch | mAP 0.375 | OpenImages mlperf validation set resized to 800x800 | [from zenodo](https://zenodo.org/record/6617981/files/resnext50_32x4d_fpn.pth) | from mlperf. [Source Code](https://github.com/mlcommons/training/tree/master/single_stage_detector/ssd/model) and [Weights](https://zenodo.org/record/6605272) | fp32 | NCHW | -| retinanet-resnext50 800x800 | onnx | mAP 0.375 | OpenImages mlperf validation set resized to 800x800 | [from zenodo](https://zenodo.org/record/6617879/files/resnext50_32x4d_fpn.onnx) | from mlperf converted from the pytorch model. [Source Code](https://github.com/mlcommons/training/tree/master/single_stage_detector/ssd/model) and [Weights](https://zenodo.org/record/6605272) | fp32 | NCHW | +| ssd-resnext50 800x800 | pytorch | mAP 0.3755 | OpenImages mlperf validation set resized to 800x800 | [from zenodo](https://zenodo.org/record/6617981/files/resnext50_32x4d_fpn.pth) | from mlperf. [Source Code](https://github.com/mlcommons/training/tree/master/single_stage_detector/ssd/model) and [Weights](https://zenodo.org/record/6605272) | fp32 | NCHW | +| ssd-resnext50 800x800 | onnx | mAP 0.3757 | OpenImages mlperf validation set resized to 800x800 | [from zenodo](https://zenodo.org/record/6617879/files/resnext50_32x4d_fpn.onnx) | from mlperf converted from the pytorch model. [Source Code](https://github.com/mlcommons/training/tree/master/single_stage_detector/ssd/model) and [Weights](https://zenodo.org/record/6605272) | fp32 | NCHW | ## Disclaimer This benchmark app is a reference implementation that is not meant to be the fastest implementation possible. From 8823573324b253e784edf29330c248c9373472e4 Mon Sep 17 00:00:00 2001 From: pgmpablo157321 Date: Tue, 28 Jun 2022 14:26:10 -0500 Subject: [PATCH 4/8] Rename resnext50 to retinanet --- tools/submission/submission-checker.py | 19 ++++++------ vision/classification_and_detection/README.md | 4 +-- .../python/dataset.py | 2 +- .../python/main.py | 30 +++++++++---------- .../python/openimages.py | 6 ++-- .../run_common.sh | 12 ++++---- ...o_onnx.md => retinanet_pytorch_to_onnx.md} | 6 ++-- ...o_onnx.py => retinanet_pytorch_to_onnx.py} | 0 8 files changed, 39 insertions(+), 40 deletions(-) rename vision/classification_and_detection/tools/{resnext_pytorch_to_onnx.md => retinanet_pytorch_to_onnx.md} (57%) rename vision/classification_and_detection/tools/{resnext_pytorch_to_onnx.py => retinanet_pytorch_to_onnx.py} (100%) diff --git a/tools/submission/submission-checker.py b/tools/submission/submission-checker.py index b8ee91618f..61011ef51c 100755 --- a/tools/submission/submission-checker.py +++ b/tools/submission/submission-checker.py @@ -566,14 +566,14 @@ }, "v2.1": { "models": [ - "resnet", "ssd-resnext50", "rnnt", + "resnet", "retinanet", "rnnt", "bert-99", "bert-99.9", "dlrm-99", "dlrm-99.9", "3d-unet-99", "3d-unet-99.9", ], "required-scenarios-datacenter": { "resnet": ["Server", "Offline"], - "ssd-resnext50": ["Server", "Offline"], + "retinanet": ["Server", "Offline"], "rnnt": ["Server", "Offline"], "bert-99": ["Server", "Offline"], "bert-99.9": ["Server", "Offline"], @@ -586,7 +586,7 @@ }, "required-scenarios-edge": { "resnet": ["SingleStream", "MultiStream", "Offline"], - "ssd-resnext50": ["SingleStream", "MultiStream", "Offline"], + "retinanet": ["SingleStream", "MultiStream", "Offline"], "rnnt": ["SingleStream", "Offline"], "bert-99": ["SingleStream", "Offline"], "3d-unet-99": ["SingleStream", "Offline"], @@ -596,7 +596,7 @@ }, "required-scenarios-datacenter-edge": { "resnet": ["SingleStream", "Offline", "MultiStream", "Server"], - "ssd-resnext50": ["SingleStream", "Offline", "MultiStream", "Server"], + "retinanet": ["SingleStream", "Offline", "MultiStream", "Server"], "rnnt": ["SingleStream", "Offline", "Server"], "bert-99": ["SingleStream", "Offline", "Server"], "bert-99.9": ["Offline", "Server"], @@ -609,8 +609,7 @@ }, "accuracy-target": { "resnet": ("acc", 76.46 * 0.99), - # TODO: Update accuracy target for ssd-resnext50 - "ssd-resnext50": ("mAP", 37.55 * 0.99), + "retinanet": ("mAP", 37.55 * 0.99), "rnnt": ("WER", (100 - 7.452) * 0.99), "bert-99": ("F1", 90.874 * 0.99), "bert-99.9": ("F1", 90.874 * 0.999), @@ -621,8 +620,8 @@ }, "performance-sample-count": { "resnet": 1024, - # TODO: Update perf sample count for ssd-resnext50 - "ssd-resnext50": 64, + # TODO: Update perf sample count for retinanet + "retinanet": 64, "rnnt": 2513, "bert-99": 10833, "bert-99.9": 10833, @@ -671,7 +670,7 @@ ], "latency-constraint": { "resnet": {"Server": 15000000}, - "ssd-resnext50": {"Server": 100000000}, + "retinanet": {"Server": 100000000}, "rnnt": {"Server": 1000000000}, "bert-99": {"Server": 130000000}, "bert-99.9": {"Server": 130000000}, @@ -680,7 +679,7 @@ }, "min-queries": { "resnet": {"SingleStream": 1024, "MultiStream": 270336, "Server": 270336, "Offline": 1}, - "ssd-resnext50": {"SingleStream": 1024, "MultiStream": 270336, "Server": 270336, "Offline": 1}, + "retinanet": {"SingleStream": 1024, "MultiStream": 270336, "Server": 270336, "Offline": 1}, "rnnt": {"SingleStream": 1024, "Server": 270336, "Offline": 1}, "bert-99": {"SingleStream": 1024, "Server": 270336, "Offline": 1}, "bert-99.9": {"SingleStream": 1024, "Server": 270336, "Offline": 1}, diff --git a/vision/classification_and_detection/README.md b/vision/classification_and_detection/README.md index b7a1406cb1..0b451bbb2d 100755 --- a/vision/classification_and_detection/README.md +++ b/vision/classification_and_detection/README.md @@ -28,8 +28,8 @@ You can find a short tutorial how to use this benchmark [here](https://github.co | ssd-resnet34 1200x1200 (removed since mlperf-v2.1)| pytorch | mAP 0.20 | coco resized to 1200x1200 | [from zenodo](https://zenodo.org/record/3236545/files/resnet34-ssd1200.pytorch) | [from mlperf](https://github.com/mlperf/inference/tree/master/others/cloud/single_stage_detector/pytorch) | fp32 | NCHW | | ssd-resnet34 1200x1200 (removed since mlperf-v2.1) | onnx | mAP 0.20 | coco resized to 1200x1200 | from zenodo [opset-8](https://zenodo.org/record/3228411/files/resnet34-ssd1200.onnx) | [from mlperf](https://github.com/mlperf/inference/tree/master/others/cloud/single_stage_detector) converted using the these [instructions](https://github.com/BowenBao/inference/tree/master/cloud/single_stage_detector/pytorch#6-onnx) | fp32 | Converted from pytorch model. | | ssd-resnet34 1200x1200 (removed since mlperf-v2.1) | onnx | mAP 0.20 | coco resized to 1200x1200 | from zenodo [opset-11](https://zenodo.org/record/4735664/files/ssd_resnet34_mAP_20.2.onnx) | [from zenodo](https://zenodo.org/record/3345892/files/tf_ssd_resnet34_22.1.zip) converted using [this script](https://github.com/mlcommons/inference/blob/master/vision/classification_and_detection/tools/convert-to-onnx.sh) | fp32 | Converted from the tensorflow model and uses the same interface as the tensorflow model. | -| ssd-resnext50 800x800 | pytorch | mAP 0.3755 | OpenImages mlperf validation set resized to 800x800 | [from zenodo](https://zenodo.org/record/6617981/files/resnext50_32x4d_fpn.pth) | from mlperf. [Source Code](https://github.com/mlcommons/training/tree/master/single_stage_detector/ssd/model) and [Weights](https://zenodo.org/record/6605272) | fp32 | NCHW | -| ssd-resnext50 800x800 | onnx | mAP 0.3757 | OpenImages mlperf validation set resized to 800x800 | [from zenodo](https://zenodo.org/record/6617879/files/resnext50_32x4d_fpn.onnx) | from mlperf converted from the pytorch model. [Source Code](https://github.com/mlcommons/training/tree/master/single_stage_detector/ssd/model) and [Weights](https://zenodo.org/record/6605272) | fp32 | NCHW | +| retinanet 800x800 | pytorch | mAP 0.3755 | OpenImages mlperf validation set resized to 800x800 | [from zenodo](https://zenodo.org/record/6617981/files/resnext50_32x4d_fpn.pth) | from mlperf. [Source Code](https://github.com/mlcommons/training/tree/master/single_stage_detector/ssd/model) and [Weights](https://zenodo.org/record/6605272) | fp32 | NCHW | +| retinanet 800x800 | onnx | mAP 0.3757 | OpenImages mlperf validation set resized to 800x800 | [from zenodo](https://zenodo.org/record/6617879/files/resnext50_32x4d_fpn.onnx) | from mlperf converted from the pytorch model. [Source Code](https://github.com/mlcommons/training/tree/master/single_stage_detector/ssd/model) and [Weights](https://zenodo.org/record/6605272) | fp32 | NCHW | ## Disclaimer This benchmark app is a reference implementation that is not meant to be the fastest implementation possible. diff --git a/vision/classification_and_detection/python/dataset.py b/vision/classification_and_detection/python/dataset.py index bfc9b7c817..02aa654c25 100755 --- a/vision/classification_and_detection/python/dataset.py +++ b/vision/classification_and_detection/python/dataset.py @@ -273,7 +273,7 @@ def pre_process_coco_resnet34_tf(img, dims=None, need_transpose=False): return img -def pre_process_openimages_resnext50(img, dims=None, need_transpose=False): +def pre_process_openimages_retinanet(img, dims=None, need_transpose=False): img = maybe_resize(img, dims) img /= 255. # transpose if needed diff --git a/vision/classification_and_detection/python/main.py b/vision/classification_and_detection/python/main.py index 279e1afc29..7b2348b1ad 100755 --- a/vision/classification_and_detection/python/main.py +++ b/vision/classification_and_detection/python/main.py @@ -50,17 +50,17 @@ "coco-300-pt": (coco.Coco, dataset.pre_process_coco_pt_mobilenet, coco.PostProcessCocoPt(False,0.3), {"image_size": [300, 300, 3]}), - "openimages-300-resnext": - (openimages.OpenImages, dataset.pre_process_openimages_resnext50, openimages.PostProcessOpenImagesResnext(False,0.05,300,300), + "openimages-300-retinanet": + (openimages.OpenImages, dataset.pre_process_openimages_retinanet, openimages.PostProcessOpenImagesRetinanet(False,0.05,300,300), {"image_size": [300, 300, 3]}), - "openimages-800-resnext": - (openimages.OpenImages, dataset.pre_process_openimages_resnext50, openimages.PostProcessOpenImagesResnext(False,0.05,800,800), + "openimages-800-retinanet": + (openimages.OpenImages, dataset.pre_process_openimages_retinanet, openimages.PostProcessOpenImagesRetinanet(False,0.05,800,800), {"image_size": [800, 800, 3]}), - "openimages-1200-resnext": - (openimages.OpenImages, dataset.pre_process_openimages_resnext50, openimages.PostProcessOpenImagesResnext(False,0.05,1200,1200), + "openimages-1200-retinanet": + (openimages.OpenImages, dataset.pre_process_openimages_retinanet, openimages.PostProcessOpenImagesRetinanet(False,0.05,1200,1200), {"image_size": [1200, 1200, 3]}), - "openimages-800-resnext-onnx": - (openimages.OpenImages, dataset.pre_process_openimages_resnext50, openimages.PostProcessOpenImagesResnext(False,0.05,800,800,False), + "openimages-800-retinanet-onnx": + (openimages.OpenImages, dataset.pre_process_openimages_retinanet, openimages.PostProcessOpenImagesRetinanet(False,0.05,800,800,False), {"image_size": [800, 800, 3]}), "coco-1200": (coco.Coco, dataset.pre_process_coco_resnet34, coco.PostProcessCoco(), @@ -181,20 +181,20 @@ "model-name": "ssd-resnet34", }, - # ssd-resnext50 - "ssd-resnext50-pytorch": { + # retinanet + "retinanet-pytorch": { "inputs": "image", "outputs": "boxes,labels,scores", - "dataset": "openimages-800-resnext", + "dataset": "openimages-800-retinanet", "backend": "pytorch-native", - "model-name": "ssd-resnext50", + "model-name": "retinanet", }, - "ssd-resnext50-onnxruntime": { + "retinanet-onnxruntime": { "inputs": "images", "outputs": "boxes,labels,scores", - "dataset": "openimages-800-resnext-onnx", + "dataset": "openimages-800-retinanet-onnx", "backend": "onnxruntime", - "model-name": "ssd-resnext50", + "model-name": "retinanet", "max-batchsize": 1 }, } diff --git a/vision/classification_and_detection/python/openimages.py b/vision/classification_and_detection/python/openimages.py index 03fe0135ae..1e08c6f38b 100644 --- a/vision/classification_and_detection/python/openimages.py +++ b/vision/classification_and_detection/python/openimages.py @@ -223,9 +223,9 @@ def finalize(self, result_dict, ds=None, output_dir=None): result_dict["mAP"] = cocoEval.stats[0] -class PostProcessOpenImagesResnext(PostProcessOpenImages): +class PostProcessOpenImagesRetinanet(PostProcessOpenImages): """ - Post processing required by ssd-resnext50 / pytorch & onnx + Post processing required by retinanet / pytorch & onnx """ def __init__(self, use_inv_map, score_threshold, height, width, dict_format=True): """ @@ -245,7 +245,7 @@ def __init__(self, use_inv_map, score_threshold, height, width, dict_format=True def __call__(self, results, ids, expected=None, result_dict=None): if self.dict_format: # If the output of the model is in dictionary format. This happens - # for the model ssd-resnext50-pytorch + # for the model retinanet-pytorch bboxes_ = [e['boxes'].cpu() for e in results] labels_ = [e['labels'].cpu() for e in results] scores_ = [e['scores'].cpu() for e in results] diff --git a/vision/classification_and_detection/run_common.sh b/vision/classification_and_detection/run_common.sh index 08201c6913..8188f61146 100755 --- a/vision/classification_and_detection/run_common.sh +++ b/vision/classification_and_detection/run_common.sh @@ -1,7 +1,7 @@ #!/bin/bash if [ $# -lt 1 ]; then - echo "usage: $0 tf|onnxruntime|pytorch|tflite [resnet50|mobilenet|ssd-mobilenet|ssd-resnet34|ssd-resnext50] [cpu|gpu]" + echo "usage: $0 tf|onnxruntime|pytorch|tflite [resnet50|mobilenet|ssd-mobilenet|ssd-resnet34|retinanet] [cpu|gpu]" exit 1 fi if [ "x$DATA_DIR" == "x" ]; then @@ -21,7 +21,7 @@ for i in $* ; do tf|onnxruntime|tflite|pytorch) backend=$i; shift;; cpu|gpu) device=$i; shift;; gpu) device=gpu; shift;; - resnet50|mobilenet|ssd-mobilenet|ssd-resnet34|ssd-resnet34-tf|ssd-resnext50) model=$i; shift;; + resnet50|mobilenet|ssd-mobilenet|ssd-resnet34|ssd-resnet34-tf|retinanet) model=$i; shift;; esac done @@ -77,9 +77,9 @@ if [ $name == "ssd-resnet34-tf-onnxruntime" ] ; then model_path="$MODEL_DIR/ssd_resnet34_mAP_20.2.onnx" profile=ssd-resnet34-onnxruntime-tf fi -if [ $name == "ssd-resnext50-onnxruntime" ] ; then +if [ $name == "retinanet-onnxruntime" ] ; then model_path="$MODEL_DIR/resnext50_32x4d_fpn.onnx" - profile=ssd-resnext50-onnxruntime + profile=retinanet-onnxruntime fi # @@ -99,9 +99,9 @@ if [ $name == "ssd-resnet34-pytorch" ] ; then model_path="$MODEL_DIR/resnet34-ssd1200.pytorch" profile=ssd-resnet34-pytorch fi -if [ $name == "ssd-resnext50-pytorch" ] ; then +if [ $name == "retinanet-pytorch" ] ; then model_path="$MODEL_DIR/resnext50_32x4d_fpn.pth" - profile=ssd-resnext50-pytorch + profile=retinanet-pytorch fi diff --git a/vision/classification_and_detection/tools/resnext_pytorch_to_onnx.md b/vision/classification_and_detection/tools/retinanet_pytorch_to_onnx.md similarity index 57% rename from vision/classification_and_detection/tools/resnext_pytorch_to_onnx.md rename to vision/classification_and_detection/tools/retinanet_pytorch_to_onnx.md index 8f90424de2..83967a0379 100644 --- a/vision/classification_and_detection/tools/resnext_pytorch_to_onnx.md +++ b/vision/classification_and_detection/tools/retinanet_pytorch_to_onnx.md @@ -8,11 +8,11 @@ Download the weights of the desired model in pytorch format. The weights used fo ``` wget https://zenodo.org/record/6605272/files/retinanet_model_10.zip ``` -Copy the script `resnext_pytorch_to_onnx.py` into the training repository in the folder `training/single_stage_detector/ssd`. From a folder that contains both repositories you will run the following command: +Copy the script `retinanet_pytorch_to_onnx.py` into the training repository in the folder `training/single_stage_detector/ssd`. From a folder that contains both repositories you can run the following command: ``` -cp inference/vision/classification_and_detection/tools/resnext_pytorch_to_onnx.py training/single_stage_detector/ssd +cp inference/vision/classification_and_detection/tools/retinanet_pytorch_to_onnx.py training/single_stage_detector/ssd ``` Run the python script: ``` -python3 resnext_pytorch_to_onnx --weights +python3 retinanet_pytorch_to_onnx --weights ``` \ No newline at end of file diff --git a/vision/classification_and_detection/tools/resnext_pytorch_to_onnx.py b/vision/classification_and_detection/tools/retinanet_pytorch_to_onnx.py similarity index 100% rename from vision/classification_and_detection/tools/resnext_pytorch_to_onnx.py rename to vision/classification_and_detection/tools/retinanet_pytorch_to_onnx.py From e6ba9702fe0b8cdb6b20f9f18012f67953853475 Mon Sep 17 00:00:00 2001 From: nvitramble <84538536+nvitramble@users.noreply.github.com> Date: Tue, 12 Jul 2022 09:42:19 -0700 Subject: [PATCH 5/8] Add new BERT calibration dataset (#1172) Co-authored-by: rameshchukka --- calibration/SQuAD-v1.1/README.md | 7 +- ...tion.txt => bert_calibration_features.txt} | 0 .../SQuAD-v1.1/bert_calibration_qas_ids.txt | 100 ++++++++++++++++++ 3 files changed, 106 insertions(+), 1 deletion(-) rename calibration/SQuAD-v1.1/{bert-calibration.txt => bert_calibration_features.txt} (100%) create mode 100755 calibration/SQuAD-v1.1/bert_calibration_qas_ids.txt diff --git a/calibration/SQuAD-v1.1/README.md b/calibration/SQuAD-v1.1/README.md index 5f0fad6d39..85b2513d71 100644 --- a/calibration/SQuAD-v1.1/README.md +++ b/calibration/SQuAD-v1.1/README.md @@ -1 +1,6 @@ -The calibration file has 100 randomly selected samples from dev-1.1.json, which contains 10570 samples in total. +The integers in bert_calibration_features.txt correspond to 100 randomly selected indices in the list of features generated from dev-v1.1.json using [convert_examples_to_features()](https://github.com/mlcommons/inference/blob/master/language/bert/create_squad_data.py#L249) with a doc_stride of 128 and a max_seq_len of 384. + +The values in bert_calibration_qas_ids.txt correspond to 100 randomly selected qas ids in the dev-v1.1.json file. + +Please only use at most 1 calibration file from this folder for calibration. + diff --git a/calibration/SQuAD-v1.1/bert-calibration.txt b/calibration/SQuAD-v1.1/bert_calibration_features.txt similarity index 100% rename from calibration/SQuAD-v1.1/bert-calibration.txt rename to calibration/SQuAD-v1.1/bert_calibration_features.txt diff --git a/calibration/SQuAD-v1.1/bert_calibration_qas_ids.txt b/calibration/SQuAD-v1.1/bert_calibration_qas_ids.txt new file mode 100755 index 0000000000..5d6f459891 --- /dev/null +++ b/calibration/SQuAD-v1.1/bert_calibration_qas_ids.txt @@ -0,0 +1,100 @@ +573020f7b2c2fd14005688fa +56beb6f23aeaaa14008c92a1 +5737a5931c456719005744e9 +5725d79e89a1e219009abf91 +56e0d9e0231d4119001ac43f +57281ab63acd2414000df496 +57269fab5951b619008f780b +5726400589a1e219009ac5f0 +572fd264b2c2fd14005684aa +56f85e71a6d7ea1400e175c4 +5728804b4b864d1900164a47 +57264cac708984140094c1b4 +5726bf135951b619008f7ceb +5728848cff5b5019007da298 +572fbf21a23a5019007fc93b +5727448b5951b619008f87a1 +5729e1101d04691400779641 +56e11afbcd28a01900c675c9 +5726642f5951b619008f7159 +56e08d32231d4119001ac2b1 +57265d86f1498d1400e8dd50 +56f7eddca6d7ea1400e172d9 +56de1645cffd8e1900b4b5d1 +5726a5525951b619008f78df +56f851b1a6d7ea1400e1755e +572a18a4af94a219006aa7e2 +57286bb84b864d19001649ca +571bb2269499d21900609cab +56d7251d0d65d214001983cc +56f88eafaef2371900626194 +571cde695efbb31900334e16 +57294279af94a219006aa20a +56bec98e3aeaaa14008c9457 +57269656708984140094cb01 +56be54bdacb8001400a50323 +571c9074dd7acb1400e4c100 +56f8b4d79b226e1400dd0e78 +5710f2e2a58dae1900cd6b73 +572683e6f1498d1400e8e24e +56f7f2e0aef2371900625cb3 +572fadcbb2c2fd1400568329 +5725fabc89a1e219009ac12a +5727aa413acd2414000de924 +56e77da237bdd419002c403d +5729e2b76aef0514001550d2 +57265e11708984140094c3bd +5726bf325951b619008f7d01 +57335fcad058e614000b5973 +572663a9f1498d1400e8ddf2 +57299ec43f37b3190047850f +56f80e1daef2371900625d8d +572689b6dd62a815002e8892 +57264a74708984140094c18c +57274d1cdd62a815002e9ab2 +572871bc4b864d1900164a04 +56d7018a0d65d214001982c5 +57111713a58dae1900cd6c02 +56bebbbf3aeaaa14008c9317 +57300e2604bcaa1900d770b7 +56f8074faef2371900625d7a +5727c94bff5b5019007d954b +5727ffb5ff5b5019007d9a8d +56e75d5037bdd419002c3ef8 +57273e50dd62a815002e9a05 +5729582b1d046914007792e4 +57290ee2af94a219006aa003 +57286ec63acd2414000df9d4 +572632ceec44d21400f3dc30 +5726f635dd62a815002e9658 +572a1f086aef0514001552c2 +57269344f1498d1400e8e440 +56bec6ac3aeaaa14008c93ff +57283adcff5b5019007d9f96 +5733266d4776f41900660714 +5725d79e89a1e219009abf94 +57280f974b864d1900164372 +570960cf200fba1400367f04 +570d28bdb3d812140066d4a7 +56e1c0f6cd28a01900c67b2e +56bec3153aeaaa14008c938b +57284618ff5b5019007da0ac +571c3e8cdd7acb1400e4c0a7 +5728fb6a1d04691400778ef6 +5726ef12dd62a815002e95a0 +57296f85af94a219006aa404 +572fe288a23a5019007fcadb +5727500f708984140094dbff +572fc659b2c2fd1400568449 +570d3468b3d812140066d545 +572a07c11d046914007796d5 +56e1fc57e3433e140042322c +573098f38ab72b1400f9c5d5 +56e1b355e3433e14004230b2 +57280cac2ca10214002d9cac +57287b4a4b864d1900164a2b +56bf36b93aeaaa14008c9565 +5728202c4b864d19001644ec +5728dab94b864d1900164f99 +57376a1bc3c5551400e51ec5 +57377083c3c5551400e51ee2 From 94f921f7c97c2a4611b97667401845d2ffce1145 Mon Sep 17 00:00:00 2001 From: Pablo Gonzalez Date: Mon, 11 Jul 2022 21:58:30 -0500 Subject: [PATCH 6/8] LoadGen: Update to version number --- loadgen/setup.py | 4 ++-- loadgen/version_generator.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/loadgen/setup.py b/loadgen/setup.py index 2a845eab1a..65441adabf 100644 --- a/loadgen/setup.py +++ b/loadgen/setup.py @@ -68,13 +68,13 @@ mlperf_loadgen_module = Extension( "mlperf_loadgen", - define_macros=[("MAJOR_VERSION", "1"), ("MINOR_VERSION", "1")], + define_macros=[("MAJOR_VERSION", "2"), ("MINOR_VERSION", "1")], include_dirs=[".", "../third_party/pybind/include"], sources=mlperf_loadgen_sources, depends=mlperf_loadgen_headers) setup(name="mlperf_loadgen", - version="1.1", + version="2.1", description="MLPerf Inference LoadGen python bindings", url="https://mlperf.org", ext_modules=[mlperf_loadgen_module]) diff --git a/loadgen/version_generator.py b/loadgen/version_generator.py index 480ee36606..7d8acbc1a6 100644 --- a/loadgen/version_generator.py +++ b/loadgen/version_generator.py @@ -93,7 +93,7 @@ def generate_loadgen_version_definitions(cc_filename, loadgen_root): ofile.write("// DO NOT EDIT: Autogenerated by version_generator.py.\n\n") ofile.write("#include \n\n") ofile.write("namespace mlperf {\n\n") - ofile.write(func_def("Version", "\"2.0\"")) + ofile.write(func_def("Version", "\"2.1\"")) date_time_now_local = datetime.datetime.now().isoformat() date_time_now_utc = datetime.datetime.utcnow().isoformat() From 5dd9ed739cc8dfeae2c5d30ec06f4cb524a4dd74 Mon Sep 17 00:00:00 2001 From: Pablo Gonzalez Date: Tue, 12 Jul 2022 18:19:13 -0500 Subject: [PATCH 7/8] Update random seeds (#1179) --- mlperf.conf | 13 +++++++------ tools/submission/submission-checker.py | 7 +++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/mlperf.conf b/mlperf.conf index 0bce4eda05..70f262443a 100644 --- a/mlperf.conf +++ b/mlperf.conf @@ -15,12 +15,13 @@ rnnt.*.performance_sample_count_override = 2513 3d-unet.*.performance_sample_count_override = 0 # Set seeds. The seeds will be distributed two weeks before the submission. -# 0x5c5c885919353c12 -*.*.qsl_rng_seed = 6655344265603136530 -# 0xdc260b99a7995230 -*.*.sample_index_rng_seed = 15863379492028895792 -# 0xafbb4808bed4f58f -*.*.schedule_rng_seed = 12662793979680847247 +# 0xc63bb0c8c280fdef +*.*.qsl_rng_seed = 14284205019438841327 +# 0x39c9343412051148 +*.*.sample_index_rng_seed = 4163916728725999944 +# 0x04267cf482328355 +*.*.schedule_rng_seed = 299063814864929621 + *.SingleStream.target_latency_percentile = 90 *.SingleStream.min_duration = 600000 diff --git a/tools/submission/submission-checker.py b/tools/submission/submission-checker.py index 61011ef51c..55f5a3c42f 100755 --- a/tools/submission/submission-checker.py +++ b/tools/submission/submission-checker.py @@ -655,11 +655,10 @@ "ssd_resnet50_v1_fpn_640x640": "ssd-small", "ssd_resnet50_v1_fpn_1024x1024": "ssd-large", }, - # TODO: Update with the real v2.1 seeds. "seeds": { - "qsl_rng_seed": 6655344265603136530, - "sample_index_rng_seed": 15863379492028895792, - "schedule_rng_seed": 12662793979680847247, + "qsl_rng_seed": 14284205019438841327, + "sample_index_rng_seed": 4163916728725999944, + "schedule_rng_seed": 299063814864929621, }, "test05_seeds": { "qsl_rng_seed" : 313588358309856706, From 67eb6a0d0e067dcc314a9f60388c0ed98ee86013 Mon Sep 17 00:00:00 2001 From: "Core.Halt" <46298335+corehalt@users.noreply.github.com> Date: Tue, 19 Jul 2022 00:29:24 +0900 Subject: [PATCH 8/8] Fix regex (#1178) --- compliance/nvidia/TEST01/verify_performance.py | 8 ++++---- compliance/nvidia/TEST04/verify_performance.py | 8 ++++---- compliance/nvidia/TEST05/verify_performance.py | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/compliance/nvidia/TEST01/verify_performance.py b/compliance/nvidia/TEST01/verify_performance.py index c02e58c6ee..44fabddee0 100644 --- a/compliance/nvidia/TEST01/verify_performance.py +++ b/compliance/nvidia/TEST01/verify_performance.py @@ -51,12 +51,12 @@ def main(): continue if ref_mode == "SingleStream": - if re.match("Early stopping 90th percentile estimate", line): + if re.match(".*Early stopping 90th percentile estimate", line): ref_score = line.split(": ",1)[1].strip() continue if ref_mode == "MultiStream": - if re.match("Early stopping 99th percentile estimate", line): + if re.match(".*Early stopping 99th percentile estimate", line): ref_score = line.split(": ",1)[1].strip() continue @@ -89,12 +89,12 @@ def main(): continue if test_mode == "SingleStream": - if re.match("Early stopping 90th percentile estimate", line): + if re.match(".*Early stopping 90th percentile estimate", line): test_score = line.split(": ",1)[1].strip() continue if test_mode == "MultiStream": - if re.match("Early stopping 99th percentile estimate", line): + if re.match(".*Early stopping 99th percentile estimate", line): test_score = line.split(": ",1)[1].strip() continue diff --git a/compliance/nvidia/TEST04/verify_performance.py b/compliance/nvidia/TEST04/verify_performance.py index a3ce5c8068..0aec33ace9 100644 --- a/compliance/nvidia/TEST04/verify_performance.py +++ b/compliance/nvidia/TEST04/verify_performance.py @@ -50,13 +50,13 @@ def main(): continue if ref_mode == "SingleStream": - if re.match("Early stopping 90th percentile estimate", line): + if re.match(".*Early stopping 90th percentile estimate", line): ref_score = line.split(": ",1)[1].strip() ref_score = 1e9 / float(ref_score) continue if ref_mode == "MultiStream": - if re.match("Early stopping 99th percentile estimate", line): + if re.match(".*Early stopping 99th percentile estimate", line): ref_score = line.split(": ",1)[1].strip() ref_score = 1e9 / float(ref_score) continue @@ -90,13 +90,13 @@ def main(): continue if test_mode == "SingleStream": - if re.match("Early stopping 90th percentile estimate", line): + if re.match(".*Early stopping 90th percentile estimate", line): test_score = line.split(": ",1)[1].strip() test_score = 1e9 / float(test_score) continue if test_mode == "MultiStream": - if re.match("Early stopping 99th percentile estimate", line): + if re.match(".*Early stopping 99th percentile estimate", line): test_score = line.split(": ",1)[1].strip() test_score = 1e9 / float(test_score) continue diff --git a/compliance/nvidia/TEST05/verify_performance.py b/compliance/nvidia/TEST05/verify_performance.py index b79815e100..09c81619e5 100644 --- a/compliance/nvidia/TEST05/verify_performance.py +++ b/compliance/nvidia/TEST05/verify_performance.py @@ -51,12 +51,12 @@ def main(): continue if ref_mode == "SingleStream": - if re.match("Early stopping 90th percentile estimate", line): + if re.match(".*Early stopping 90th percentile estimate", line): ref_score = line.split(": ",1)[1].strip() continue if ref_mode == "MultiStream": - if re.match("Early stopping 99th percentile estimate", line): + if re.match(".*Early stopping 99th percentile estimate", line): ref_score = line.split(": ",1)[1].strip() continue @@ -89,12 +89,12 @@ def main(): continue if test_mode == "SingleStream": - if re.match("Early stopping 90th percentile estimate", line): + if re.match(".*Early stopping 90th percentile estimate", line): test_score = line.split(": ",1)[1].strip() continue if test_mode == "MultiStream": - if re.match("Early stopping 99th percentile estimate", line): + if re.match(".*Early stopping 99th percentile estimate", line): test_score = line.split(": ",1)[1].strip() continue