fix: address reviews from @senokay

ci-constraints.txt

@@ -0,0 +1,5 @@
+# This pip constraints file is for the reproducibility of model accuracies
+opencv-python-headless==4.8.0.76
+torch==2.0.1
+torchvision==0.15.2
+numpy==1.25.2

docker/Dockerfile

@@ -33,4 +33,4 @@ RUN --mount=type=secret,id=furiosa.conf,dst=/etc/apt/auth.conf.d/furiosa.conf,re
     apt-get update && \
     make toolchain
 RUN --mount=type=secret,id=.netrc,dst=/root/.netrc,required \
-    pip install --pre --extra-index-url https://internal-pypi.furiosa.dev/simple --constraint constraints.txt .[test]
+    pip install --pre --extra-index-url https://internal-pypi.furiosa.dev/simple --constraint ci-constraints.txt .[test]

furiosa/models/client/api.py

@@ -110,6 +110,10 @@ def run_inferences(model_cls: Type[Model], input_paths: Sequence[str], postproce
         model = model_cls(postprocessor_type=postprocess)
     else:
         model = model_cls()
+    # FIXME: For native postprocess implementations, only YOLO can handle multiple contexts
+    single_context = not isinstance(model.postprocessor, PythonPostProcessor) and not isinstance(
+        model, (vision.YOLOv5m, vision.YOLOv5l)
+    )
     queries = len(input_paths)
     print(f"Running {queries} input samples ...")
     print(decorate_with_bar(warning))
@@ -124,10 +128,6 @@ def run_inferences(model_cls: Type[Model], input_paths: Sequence[str], postproce
         after_npu = perf_counter()
         for contexted_model_output in tqdm(model_outputs, desc="Postprocess"):
             model_output, context = contexted_model_output
-            # FIXME: Only YOLO can handle multiple contexts
-            single_context = not isinstance(
-                model.postprocessor, PythonPostProcessor
-            ) and not isinstance(model, (vision.YOLOv5m, vision.YOLOv5l))
             context = context[0] if context is not None and single_context else context
             model.postprocess(model_output, context)
         all_done = perf_counter()

tests/bench/test_efficientnet_b0.py

@@ -46,9 +46,8 @@ def workload(image, answer):
         else:
             incorrect_predictions += 1
 
-    runner = create_runner(model.model_source())
-    benchmark.pedantic(workload, setup=read_image, rounds=num_images)
-    runner.close()
+    with create_runner(model.model_source()) as runner:
+        benchmark.pedantic(workload, setup=read_image, rounds=num_images)
 
     total_predictions = correct_predictions + incorrect_predictions
     accuracy = 100.0 * correct_predictions / total_predictions

tests/bench/test_efficientnet_v2_s.py

@@ -46,9 +46,8 @@ def workload(image, answer):
         else:
             incorrect_predictions += 1
 
-    runner = create_runner(model.model_source())
-    benchmark.pedantic(workload, setup=read_image, rounds=num_images)
-    runner.close()
+    with create_runner(model.model_source()) as runner:
+        benchmark.pedantic(workload, setup=read_image, rounds=num_images)
 
     total_predictions = correct_predictions + incorrect_predictions
     accuracy = 100.0 * correct_predictions / total_predictions

tests/bench/test_resnet50.py

@@ -47,9 +47,8 @@ def workload(image, answer):
         else:
             incorrect_predictions += 1
 
-    runner = create_runner(model.model_source())
-    benchmark.pedantic(workload, setup=read_image, rounds=num_images)
-    runner.close()
+    with create_runner(model.model_source()) as runner:
+        benchmark.pedantic(workload, setup=read_image, rounds=num_images)
 
     total_predictions = correct_predictions + incorrect_predictions
     accuracy = 100.0 * correct_predictions / total_predictions

tests/bench/test_ssd_mobilenet.py

@@ -110,9 +110,8 @@ def workload(image_id, image):
             }
             detections.append(detection)
 
-    runner = create_runner(model.model_source())
-    benchmark.pedantic(workload, setup=read_image, rounds=num_images)
-    runner.close()
+    with create_runner(model.model_source()) as runner:
+        benchmark.pedantic(workload, setup=read_image, rounds=num_images)
 
     coco_detections = coco.loadRes(detections)
     coco_eval = COCOeval(coco, coco_detections, iouType="bbox")

tests/bench/test_ssd_resnet34.py

@@ -70,9 +70,8 @@ def workload(image_id, image):
             }
             detections.append(detection)
 
-    runner = create_runner(model.model_source())
-    benchmark.pedantic(workload, setup=read_image, rounds=num_images)
-    runner.close()
+    with create_runner(model.model_source()) as runner:
+        benchmark.pedantic(workload, setup=read_image, rounds=num_images)
 
     coco_detections = coco.loadRes(detections)
     coco_eval = COCOeval(coco, coco_detections, iouType="bbox")
@@ -126,9 +125,8 @@ def workload(image_id, image):
             }
             detections.append(detection)
 
-    runner = create_runner(model.model_source())
-    benchmark.pedantic(workload, setup=read_image, rounds=num_images)
-    runner.close()
+    with create_runner(model.model_source()) as runner:
+        benchmark.pedantic(workload, setup=read_image, rounds=num_images)
 
     coco_detections = coco.loadRes(detections)
     coco_eval = COCOeval(coco, coco_detections, iouType="bbox")

tests/bench/test_yolov5l.py

@@ -56,9 +56,8 @@ def workload(im, boxes_target, classes_target):
             classes_target=classes_target,
         )
 
-    runner = create_runner(model.model_source())
-    benchmark.pedantic(workload, setup=read_image, rounds=num_images)
-    runner.close()
+    with create_runner(model.model_source()) as runner:
+        benchmark.pedantic(workload, setup=read_image, rounds=num_images)
 
     result = metric.compute()
     print("YOLOv5Large mAP:", result['map'])

tests/bench/test_yolov5m.py

@@ -58,9 +58,8 @@ def workload(im, boxes_target, classes_target):
             classes_target=classes_target,
         )
 
-    runner = create_runner(model.model_source())
-    benchmark.pedantic(workload, setup=read_image, rounds=num_images)
-    runner.close()
+    with create_runner(model.model_source()) as runner:
+        benchmark.pedantic(workload, setup=read_image, rounds=num_images)
 
     result = metric.compute()
     print("YOLOv5Medium mAP:", result['map'])