From b850c4732e7e5681887f1e58289a489376a38f67 Mon Sep 17 00:00:00 2001 From: Myeong-geun Shin Date: Sat, 12 Aug 2023 02:39:17 +0900 Subject: [PATCH] fix: address reviews from @senokay --- ci-constraints.txt | 5 +++++ constraints.txt | 4 ---- docker/Dockerfile | 2 +- furiosa/models/client/api.py | 8 ++++---- tests/bench/test_efficientnet_b0.py | 5 ++--- tests/bench/test_efficientnet_v2_s.py | 5 ++--- tests/bench/test_resnet50.py | 5 ++--- tests/bench/test_ssd_mobilenet.py | 5 ++--- tests/bench/test_ssd_resnet34.py | 10 ++++------ tests/bench/test_yolov5l.py | 5 ++--- tests/bench/test_yolov5m.py | 5 ++--- 11 files changed, 26 insertions(+), 33 deletions(-) create mode 100644 ci-constraints.txt delete mode 100644 constraints.txt diff --git a/ci-constraints.txt b/ci-constraints.txt new file mode 100644 index 00000000..45e95128 --- /dev/null +++ b/ci-constraints.txt @@ -0,0 +1,5 @@ +# This pip constraints file is for the reproducibility of model accuracies +opencv-python-headless==4.8.0.76 +torch==2.0.1 +torchvision==0.15.2 +numpy==1.25.2 diff --git a/constraints.txt b/constraints.txt deleted file mode 100644 index 0b97f1ed..00000000 --- a/constraints.txt +++ /dev/null @@ -1,4 +0,0 @@ -opencv-python-headless==4.8.0.76 -torch==2.0.1 -torchvision==0.15.2 -numpy==1.25.2 \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 76a396cc..7a0ad2df 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -33,4 +33,4 @@ RUN --mount=type=secret,id=furiosa.conf,dst=/etc/apt/auth.conf.d/furiosa.conf,re apt-get update && \ make toolchain RUN --mount=type=secret,id=.netrc,dst=/root/.netrc,required \ - pip install --pre --extra-index-url https://internal-pypi.furiosa.dev/simple --constraint constraints.txt .[test] + pip install --pre --extra-index-url https://internal-pypi.furiosa.dev/simple --constraint ci-constraints.txt .[test] diff --git a/furiosa/models/client/api.py b/furiosa/models/client/api.py index 79ffc3dc..6236ae57 100644 --- a/furiosa/models/client/api.py +++ b/furiosa/models/client/api.py @@ -110,6 +110,10 @@ def run_inferences(model_cls: Type[Model], input_paths: Sequence[str], postproce model = model_cls(postprocessor_type=postprocess) else: model = model_cls() + # FIXME: For native postprocess implementations, only YOLO can handle multiple contexts + single_context = not isinstance(model.postprocessor, PythonPostProcessor) and not isinstance( + model, (vision.YOLOv5m, vision.YOLOv5l) + ) queries = len(input_paths) print(f"Running {queries} input samples ...") print(decorate_with_bar(warning)) @@ -124,10 +128,6 @@ def run_inferences(model_cls: Type[Model], input_paths: Sequence[str], postproce after_npu = perf_counter() for contexted_model_output in tqdm(model_outputs, desc="Postprocess"): model_output, context = contexted_model_output - # FIXME: Only YOLO can handle multiple contexts - single_context = not isinstance( - model.postprocessor, PythonPostProcessor - ) and not isinstance(model, (vision.YOLOv5m, vision.YOLOv5l)) context = context[0] if context is not None and single_context else context model.postprocess(model_output, context) all_done = perf_counter() diff --git a/tests/bench/test_efficientnet_b0.py b/tests/bench/test_efficientnet_b0.py index 9fb3af31..1faef857 100644 --- a/tests/bench/test_efficientnet_b0.py +++ b/tests/bench/test_efficientnet_b0.py @@ -46,9 +46,8 @@ def workload(image, answer): else: incorrect_predictions += 1 - runner = create_runner(model.model_source()) - benchmark.pedantic(workload, setup=read_image, rounds=num_images) - runner.close() + with create_runner(model.model_source()) as runner: + benchmark.pedantic(workload, setup=read_image, rounds=num_images) total_predictions = correct_predictions + incorrect_predictions accuracy = 100.0 * correct_predictions / total_predictions diff --git a/tests/bench/test_efficientnet_v2_s.py b/tests/bench/test_efficientnet_v2_s.py index 6a9f658c..e4cbd6d5 100644 --- a/tests/bench/test_efficientnet_v2_s.py +++ b/tests/bench/test_efficientnet_v2_s.py @@ -46,9 +46,8 @@ def workload(image, answer): else: incorrect_predictions += 1 - runner = create_runner(model.model_source()) - benchmark.pedantic(workload, setup=read_image, rounds=num_images) - runner.close() + with create_runner(model.model_source()) as runner: + benchmark.pedantic(workload, setup=read_image, rounds=num_images) total_predictions = correct_predictions + incorrect_predictions accuracy = 100.0 * correct_predictions / total_predictions diff --git a/tests/bench/test_resnet50.py b/tests/bench/test_resnet50.py index 4b3b6b1f..4c8d4841 100644 --- a/tests/bench/test_resnet50.py +++ b/tests/bench/test_resnet50.py @@ -47,9 +47,8 @@ def workload(image, answer): else: incorrect_predictions += 1 - runner = create_runner(model.model_source()) - benchmark.pedantic(workload, setup=read_image, rounds=num_images) - runner.close() + with create_runner(model.model_source()) as runner: + benchmark.pedantic(workload, setup=read_image, rounds=num_images) total_predictions = correct_predictions + incorrect_predictions accuracy = 100.0 * correct_predictions / total_predictions diff --git a/tests/bench/test_ssd_mobilenet.py b/tests/bench/test_ssd_mobilenet.py index c66289d0..da0e7e27 100644 --- a/tests/bench/test_ssd_mobilenet.py +++ b/tests/bench/test_ssd_mobilenet.py @@ -110,9 +110,8 @@ def workload(image_id, image): } detections.append(detection) - runner = create_runner(model.model_source()) - benchmark.pedantic(workload, setup=read_image, rounds=num_images) - runner.close() + with create_runner(model.model_source()) as runner: + benchmark.pedantic(workload, setup=read_image, rounds=num_images) coco_detections = coco.loadRes(detections) coco_eval = COCOeval(coco, coco_detections, iouType="bbox") diff --git a/tests/bench/test_ssd_resnet34.py b/tests/bench/test_ssd_resnet34.py index 3d32af76..ff56a014 100644 --- a/tests/bench/test_ssd_resnet34.py +++ b/tests/bench/test_ssd_resnet34.py @@ -70,9 +70,8 @@ def workload(image_id, image): } detections.append(detection) - runner = create_runner(model.model_source()) - benchmark.pedantic(workload, setup=read_image, rounds=num_images) - runner.close() + with create_runner(model.model_source()) as runner: + benchmark.pedantic(workload, setup=read_image, rounds=num_images) coco_detections = coco.loadRes(detections) coco_eval = COCOeval(coco, coco_detections, iouType="bbox") @@ -126,9 +125,8 @@ def workload(image_id, image): } detections.append(detection) - runner = create_runner(model.model_source()) - benchmark.pedantic(workload, setup=read_image, rounds=num_images) - runner.close() + with create_runner(model.model_source()) as runner: + benchmark.pedantic(workload, setup=read_image, rounds=num_images) coco_detections = coco.loadRes(detections) coco_eval = COCOeval(coco, coco_detections, iouType="bbox") diff --git a/tests/bench/test_yolov5l.py b/tests/bench/test_yolov5l.py index 6adf5ac2..73ef9736 100644 --- a/tests/bench/test_yolov5l.py +++ b/tests/bench/test_yolov5l.py @@ -56,9 +56,8 @@ def workload(im, boxes_target, classes_target): classes_target=classes_target, ) - runner = create_runner(model.model_source()) - benchmark.pedantic(workload, setup=read_image, rounds=num_images) - runner.close() + with create_runner(model.model_source()) as runner: + benchmark.pedantic(workload, setup=read_image, rounds=num_images) result = metric.compute() print("YOLOv5Large mAP:", result['map']) diff --git a/tests/bench/test_yolov5m.py b/tests/bench/test_yolov5m.py index 4f64ee13..4f763fcc 100644 --- a/tests/bench/test_yolov5m.py +++ b/tests/bench/test_yolov5m.py @@ -58,9 +58,8 @@ def workload(im, boxes_target, classes_target): classes_target=classes_target, ) - runner = create_runner(model.model_source()) - benchmark.pedantic(workload, setup=read_image, rounds=num_images) - runner.close() + with create_runner(model.model_source()) as runner: + benchmark.pedantic(workload, setup=read_image, rounds=num_images) result = metric.compute() print("YOLOv5Medium mAP:", result['map'])