Use ophyd_async OAV (#594)

* Fix imports in i24

* Pin to dodal

* A first pass replacing oav with ophyd_async device

* Update dodal pin

* Fix linting

* Try to fix thawing pla

* Still ignoring mock device

* Get thawing tests to pass

* Pin dodal

* Start fixing some tests

* Try to fix some more tests

* Let's see what fails now

* Fix a couple more

* Just a handful left

* Fix mock oav in simple_beamline

* Finish fixing tests

* Remove assert

* Update dodal pin

pyproject.toml

@@ -46,7 +46,7 @@ dependencies = [
     "ophyd == 1.9.0",
     "ophyd-async >= 0.3a5",
     "bluesky >= 1.13.0a4",
-    "dls-dodal @ git+https://github.com/DiamondLightSource/dodal.git@d27d1a4d05192d005c3fab8c229995eeba96e16f",
+    "dls-dodal @ git+https://github.com/DiamondLightSource/dodal.git@598d2560e5bdb1f92c9f4895563274449233a6a8",
 ]
 
 

src/mx_bluesky/beamlines/i04/thawing_plan.py

@@ -37,13 +37,18 @@ def switch_forwarder_to_ROI() -> MsgGenerator:
         yield from bps.mv(oav_to_redis_forwarder.selected_source, Source.ROI)  # type: ignore # See: https://github.com/bluesky/bluesky/issues/1809
         yield from bps.kickoff(oav_to_redis_forwarder, wait=True)
 
+    microns_per_pixel_x = yield from bps.rd(oav.microns_per_pixel_x)
+    microns_per_pixel_y = yield from bps.rd(oav.microns_per_pixel_y)
+    beam_centre_i = yield from bps.rd(oav.beam_centre_i)
+    beam_centre_j = yield from bps.rd(oav.beam_centre_j)
+
     @subs_decorator(MurkoCallback(REDIS_HOST, REDIS_PASSWORD, MURKO_REDIS_DB))
     @run_decorator(
         md={
-            "microns_per_x_pixel": oav.parameters.micronsPerXPixel,
-            "microns_per_y_pixel": oav.parameters.micronsPerYPixel,
-            "beam_centre_i": oav.parameters.beam_centre_i,
-            "beam_centre_j": oav.parameters.beam_centre_j,
+            "microns_per_x_pixel": microns_per_pixel_x,
+            "microns_per_y_pixel": microns_per_pixel_y,
+            "beam_centre_i": beam_centre_i,
+            "beam_centre_j": beam_centre_j,
             "zoom_percentage": zoom_percentage,
             "sample_id": sample_id,
         }

src/mx_bluesky/beamlines/i24/serial/fixed_target/i24ssx_moveonclick.py

@@ -11,7 +11,7 @@
 from bluesky.run_engine import RunEngine
 from dodal.beamlines import i24
 from dodal.devices.i24.pmac import PMAC
-from dodal.devices.oav.oav_async import OAV
+from dodal.devices.oav.oav_detector import OAV
 
 from mx_bluesky.beamlines.i24.serial.fixed_target import (
     i24ssx_Chip_Manager_py3v1 as manager,

src/mx_bluesky/hyperion/device_setup_plans/setup_oav.py

@@ -1,11 +1,10 @@
 from functools import partial
 
 import bluesky.plan_stubs as bps
+from dodal.devices.areadetector.plugins.CAM import ColorMode
 from dodal.devices.oav.oav_detector import OAV
-from dodal.devices.oav.oav_errors import OAVError_ZoomLevelNotFound
 from dodal.devices.oav.oav_parameters import OAVParameters
 from dodal.devices.oav.pin_image_recognition import PinTipDetection
-from dodal.devices.oav.utils import ColorMode
 
 from mx_bluesky.hyperion.parameters.constants import CONST
 
@@ -62,11 +61,6 @@ def setup_general_oav_params(oav: OAV, parameters: OAVParameters):
     yield from set_using_group(oav.cam.gain, parameters.gain)  # type: ignore # See: https://github.com/bluesky/bluesky/issues/1809
 
     zoom_level_str = f"{float(parameters.zoom)}x"
-    if zoom_level_str not in oav.zoom_controller.allowed_zoom_levels:
-        raise OAVError_ZoomLevelNotFound(
-            f"Found {zoom_level_str} as a zoom level but expected one of {oav.zoom_controller.allowed_zoom_levels}"
-        )
-
     yield from bps.abs_set(
         oav.zoom_controller,  # type: ignore # See: https://github.com/bluesky/bluesky/issues/1809
         zoom_level_str,

src/mx_bluesky/hyperion/experiment_plans/grid_detect_then_xray_centre_plan.py

@@ -110,7 +110,7 @@ def detect_grid_and_do_gridscan(
 ):
     snapshot_template = f"{parameters.detector_params.prefix}_{parameters.detector_params.run_number}_{{angle}}"
 
-    grid_params_callback = GridDetectionCallback(composite.oav.parameters)
+    grid_params_callback = GridDetectionCallback()
 
     @bpp.subs_decorator([grid_params_callback])
     def run_grid_detection_plan(

src/mx_bluesky/hyperion/experiment_plans/oav_grid_detection_plan.py

@@ -90,12 +90,13 @@ def grid_detection_plan(
 
     LOGGER.info("OAV Centring: Camera set up")
 
-    assert isinstance(oav.parameters.micronsPerXPixel, float)
-    box_size_x_pixels = box_size_um / oav.parameters.micronsPerXPixel
-    assert isinstance(oav.parameters.micronsPerYPixel, float)
-    box_size_y_pixels = box_size_um / oav.parameters.micronsPerYPixel
+    microns_per_pixel_x = yield from bps.rd(oav.microns_per_pixel_x)
+    microns_per_pixel_y = yield from bps.rd(oav.microns_per_pixel_y)
 
-    grid_width_pixels = int(grid_width_microns / oav.parameters.micronsPerXPixel)
+    box_size_x_pixels = box_size_um / microns_per_pixel_x
+    box_size_y_pixels = box_size_um / microns_per_pixel_y
+
+    grid_width_pixels = int(grid_width_microns / microns_per_pixel_x)
 
     # The FGS uses -90 so we need to match it
     for angle in [0, -90]:
@@ -115,7 +116,7 @@ def grid_detection_plan(
             (yield from bps.rd(pin_tip_detection.triggered_bottom_edge))
         )
 
-        full_image_height_px = yield from bps.rd(oav.cam.array_size.array_size_y)
+        full_image_height_px = yield from bps.rd(oav.cam.array_size_y)
 
         # only use the area from the start of the pin onwards
         top_edge = top_edge[tip_x_px : tip_x_px + grid_width_pixels]
@@ -164,7 +165,7 @@ def grid_detection_plan(
         yield from bps.trigger(oav.grid_snapshot, wait=True)  # type: ignore # See: https://github.com/bluesky/bluesky/issues/1809
         yield from bps.create(CONST.DESCRIPTORS.OAV_GRID_SNAPSHOT_TRIGGERED)
 
-        yield from bps.read(oav.grid_snapshot)  # type: ignore # See: https://github.com/bluesky/bluesky/issues/1809
+        yield from bps.read(oav)  # type: ignore # See: https://github.com/bluesky/bluesky/issues/1809
         yield from bps.read(smargon)
         yield from bps.save()
 

src/mx_bluesky/hyperion/experiment_plans/pin_tip_centring_plan.py

@@ -132,13 +132,13 @@ def pin_tip_centre_plan(
     pin_tip_setup = composite.pin_tip_detection
     pin_tip_detect = composite.pin_tip_detection
 
-    assert oav.parameters.micronsPerXPixel is not None
-    tip_offset_px = int(tip_offset_microns / oav.parameters.micronsPerXPixel)
+    microns_per_pixel_x = yield from bps.rd(oav.microns_per_pixel_x)
+    tip_offset_px = int(tip_offset_microns / microns_per_pixel_x)
 
     def offset_and_move(tip: Pixel):
         pixel_to_move_to = (tip[0] + tip_offset_px, tip[1])
         position_mm = yield from get_move_required_so_that_beam_is_at_pixel(
-            smargon, pixel_to_move_to, oav.parameters
+            smargon, pixel_to_move_to, oav
         )
         LOGGER.info(f"Tip centring moving to : {position_mm}")
         yield from move_smargon_warn_on_out_of_range(smargon, position_mm)

src/mx_bluesky/hyperion/external_interaction/callbacks/grid_detection_callback.py

@@ -2,7 +2,6 @@
 
 import numpy as np
 from bluesky.callbacks import CallbackBase
-from dodal.devices.oav.oav_detector import OAVConfigParams
 from dodal.devices.oav.utils import calculate_x_y_z_of_pixel
 from event_model.documents import Event
 
@@ -26,21 +25,19 @@ class GridParamUpdate(TypedDict):
 class GridDetectionCallback(CallbackBase):
     def __init__(
         self,
-        oav_params: OAVConfigParams,
         *args,
     ) -> None:
         super().__init__(*args)
-        self.oav_params = oav_params
         self.start_positions: list = []
         self.box_numbers: list = []
 
     def event(self, doc: Event):
         data = doc.get("data")
-        top_left_x_px = data["oav_grid_snapshot_top_left_x"]
-        box_width_px = data["oav_grid_snapshot_box_width"]
+        top_left_x_px = data["oav-grid_snapshot-top_left_x"]
+        box_width_px = data["oav-grid_snapshot-box_width"]
         x_of_centre_of_first_box_px = top_left_x_px + box_width_px / 2
 
-        top_left_y_px = data["oav_grid_snapshot_top_left_y"]
+        top_left_y_px = data["oav-grid_snapshot-top_left_y"]
         y_of_centre_of_first_box_px = top_left_y_px + box_width_px / 2
 
         smargon_omega = data["smargon-omega"]
@@ -53,23 +50,32 @@ def event(self, doc: Event):
             y_of_centre_of_first_box_px,
         )
 
+        microns_per_pixel_x = data["oav-microns_per_pixel_x"]
+        microns_per_pixel_y = data["oav-microns_per_pixel_y"]
+        beam_x = data["oav-snapshot-beam_centre_i"]
+        beam_y = data["oav-snapshot-beam_centre_j"]
+
         position_grid_start = calculate_x_y_z_of_pixel(
-            current_xyz, smargon_omega, centre_of_first_box, self.oav_params
+            current_xyz,
+            smargon_omega,
+            centre_of_first_box,
+            (beam_x, beam_y),
+            (microns_per_pixel_x, microns_per_pixel_y),
         )
 
         LOGGER.info(f"Calculated start position {position_grid_start}")
 
         self.start_positions.append(position_grid_start)
         self.box_numbers.append(
             (
-                data["oav_grid_snapshot_num_boxes_x"],
-                data["oav_grid_snapshot_num_boxes_y"],
+                data["oav-grid_snapshot-num_boxes_x"],
+                data["oav-grid_snapshot-num_boxes_y"],
             )
         )
 
-        self.x_step_size_mm = box_width_px * self.oav_params.micronsPerXPixel / 1000
-        self.y_step_size_mm = box_width_px * self.oav_params.micronsPerYPixel / 1000
-        self.z_step_size_mm = box_width_px * self.oav_params.micronsPerYPixel / 1000
+        self.x_step_size_mm = box_width_px * microns_per_pixel_x / 1000
+        self.y_step_size_mm = box_width_px * microns_per_pixel_y / 1000
+        self.z_step_size_mm = box_width_px * microns_per_pixel_y / 1000
         return doc
 
     def get_grid_parameters(self) -> GridParamUpdate:

src/mx_bluesky/hyperion/external_interaction/callbacks/robot_load/ispyb_callback.py

@@ -62,7 +62,7 @@ def activity_gated_event(self, doc: Event) -> Event | None:
                 self.action_id is not None
             ), "ISPyB Robot load callback event called unexpectedly"
             barcode = doc["data"]["robot-barcode"]
-            oav_snapshot = doc["data"]["oav_snapshot_last_saved_path"]
+            oav_snapshot = doc["data"]["oav-snapshot-last_saved_path"]
             webcam_snapshot = doc["data"]["webcam-last_saved_path"]
             # I03 uses webcam/oav snapshots in place of before/after snapshots
             self.expeye.update_barcode_and_snapshots(

src/mx_bluesky/hyperion/external_interaction/callbacks/rotation/ispyb_callback.py

@@ -158,7 +158,7 @@ def _handle_oav_rotation_snapshot_triggered(self, doc) -> Sequence[ScanDataInfo]
         data_collection_info = DataCollectionInfo(
             **{
                 f"xtal_snapshot{self._oav_snapshot_event_idx}": data.get(
-                    "oav_snapshot_last_saved_path"
+                    "oav-snapshot-last_saved_path"
                 )
             }
         )

src/mx_bluesky/hyperion/external_interaction/callbacks/xray_centre/ispyb_callback.py

@@ -180,25 +180,25 @@ def _handle_oav_grid_snapshot_triggered(self, doc) -> Sequence[ScanDataInfo]:
         data = doc["data"]
         data_collection_id = None
         data_collection_info = DataCollectionInfo(
-            xtal_snapshot1=data.get("oav_grid_snapshot_last_path_full_overlay"),
-            xtal_snapshot2=data.get("oav_grid_snapshot_last_path_outer"),
-            xtal_snapshot3=data.get("oav_grid_snapshot_last_saved_path"),
+            xtal_snapshot1=data.get("oav-grid_snapshot-last_path_full_overlay"),
+            xtal_snapshot2=data.get("oav-grid_snapshot-last_path_outer"),
+            xtal_snapshot3=data.get("oav-grid_snapshot-last_saved_path"),
             n_images=(
-                data["oav_grid_snapshot_num_boxes_x"]
-                * data["oav_grid_snapshot_num_boxes_y"]
+                data["oav-grid_snapshot-num_boxes_x"]
+                * data["oav-grid_snapshot-num_boxes_y"]
             ),
         )
-        microns_per_pixel_x = data["oav_grid_snapshot_microns_per_pixel_x"]
-        microns_per_pixel_y = data["oav_grid_snapshot_microns_per_pixel_y"]
+        microns_per_pixel_x = data["oav-microns_per_pixel_x"]
+        microns_per_pixel_y = data["oav-microns_per_pixel_y"]
         data_collection_grid_info = DataCollectionGridInfo(
-            dx_in_mm=data["oav_grid_snapshot_box_width"] * microns_per_pixel_x / 1000,
-            dy_in_mm=data["oav_grid_snapshot_box_width"] * microns_per_pixel_y / 1000,
-            steps_x=data["oav_grid_snapshot_num_boxes_x"],
-            steps_y=data["oav_grid_snapshot_num_boxes_y"],
+            dx_in_mm=data["oav-grid_snapshot-box_width"] * microns_per_pixel_x / 1000,
+            dy_in_mm=data["oav-grid_snapshot-box_width"] * microns_per_pixel_y / 1000,
+            steps_x=data["oav-grid_snapshot-num_boxes_x"],
+            steps_y=data["oav-grid_snapshot-num_boxes_y"],
             microns_per_pixel_x=microns_per_pixel_x,
             microns_per_pixel_y=microns_per_pixel_y,
-            snapshot_offset_x_pixel=int(data["oav_grid_snapshot_top_left_x"]),
-            snapshot_offset_y_pixel=int(data["oav_grid_snapshot_top_left_y"]),
+            snapshot_offset_x_pixel=int(data["oav-grid_snapshot-top_left_x"]),
+            snapshot_offset_y_pixel=int(data["oav-grid_snapshot-top_left_y"]),
             orientation=Orientation.HORIZONTAL,
             snaked=True,
         )

src/mx_bluesky/hyperion/utils/validation.py

@@ -8,7 +8,7 @@
 import bluesky.preprocessors as bpp
 from bluesky.run_engine import RunEngine
 from dodal.beamlines import i03
-from dodal.devices.oav.oav_parameters import OAVConfigParams
+from dodal.devices.oav.oav_parameters import OAVConfig
 from ophyd_async.core import set_mock_value
 
 from mx_bluesky.hyperion.device_setup_plans.read_hardware_for_setup import (
@@ -23,8 +23,8 @@
 from mx_bluesky.hyperion.parameters.constants import CONST
 from mx_bluesky.hyperion.parameters.rotation import RotationScan
 
-DISPLAY_CONFIGURATION = "tests/devices/unit_tests/test_display.configuration"
-ZOOM_LEVELS_XML = "tests/devices/unit_tests/test_jCameraManZoomLevels.xml"
+DISPLAY_CONFIGURATION = "tests/test_data/test_display.configuration"
+ZOOM_LEVELS_XML = "tests/test_data/test_jCameraManZoomLevels.xml"
 TEST_DATA_DIRECTORY = Path("tests/test_data/nexus_files/rotation")
 TEST_METAFILE = "ins_8_5_meta.h5.gz"
 FAKE_DATAFILE = "../fake_data.h5"
@@ -94,15 +94,14 @@ def fake_create_rotation_devices():
     robot = i03.robot(fake_with_ophyd_sim=True)
     oav = i03.oav(
         fake_with_ophyd_sim=True,
-        params=OAVConfigParams(
-            zoom_params_file=ZOOM_LEVELS_XML, display_config=DISPLAY_CONFIGURATION
+        params=OAVConfig(
+            zoom_params_file=ZOOM_LEVELS_XML, display_config_file=DISPLAY_CONFIGURATION
         ),
     )
     xbpm_feedback = i03.xbpm_feedback(fake_with_ophyd_sim=True)
 
     set_mock_value(smargon.omega.max_velocity, 131)
     set_mock_value(dcm.energy_in_kev.user_readback, 12700)
-    oav.zoom_controller.fvst.sim_put("1.0x")  # type: ignore
 
     return RotationScanComposite(
         attenuator=attenuator,

tests/conftest.py

@@ -8,7 +8,7 @@
 from contextlib import ExitStack
 from functools import partial
 from typing import Any
-from unittest.mock import MagicMock, patch
+from unittest.mock import AsyncMock, MagicMock, patch
 
 import bluesky.plan_stubs as bps
 import numpy
@@ -22,7 +22,7 @@
 from dodal.common.beamlines.beamline_parameters import (
     GDABeamlineParameters,
 )
-from dodal.common.beamlines.beamline_utils import clear_device, clear_devices
+from dodal.common.beamlines.beamline_utils import clear_devices
 from dodal.devices.aperturescatterguard import (
     AperturePosition,
     ApertureScatterguard,
@@ -35,7 +35,7 @@
 from dodal.devices.eiger import EigerDetector
 from dodal.devices.fast_grid_scan import FastGridScanCommon
 from dodal.devices.flux import Flux
-from dodal.devices.oav.oav_detector import OAV, OAVConfigParams
+from dodal.devices.oav.oav_detector import OAV, OAVConfig
 from dodal.devices.oav.oav_parameters import OAVParameters
 from dodal.devices.robot import BartRobot
 from dodal.devices.s4_slit_gaps import S4SlitGaps
@@ -351,38 +351,23 @@ def synchrotron(RE):
 
 
 @pytest.fixture
-def oav(test_config_files):
-    parameters = OAVConfigParams(
+def oav(test_config_files, RE):
+    parameters = OAVConfig(
         test_config_files["zoom_params_file"], test_config_files["display_config"]
     )
-    parameters.micronsPerXPixel = 2.87
-    parameters.micronsPerYPixel = 2.87
-
-    # This should only be needed until issues with https://github.com/DiamondLightSource/dodal/pull/854
-    # or ophyd-async OAV are resolved
-    try:
-        clear_device("oav")
-    except KeyError:
-        ...
     oav = i03.oav(fake_with_ophyd_sim=True, params=parameters)
 
-    oav.zoom_controller.zrst.set("1.0x")
-    oav.zoom_controller.onst.set("2.0x")
+    zoom_levels_list = ["1.0x", "3.0x", "5.0x", "7.5x", "10.0x", "15.0x"]
+    oav.zoom_controller._get_allowed_zoom_levels = AsyncMock(
+        return_value=zoom_levels_list
+    )
+    # Equivalent to previously set values for microns and beam centre
+    set_mock_value(oav.zoom_controller.level, "5.0x")
 
-    oav.parameters.micronsPerXPixel = 1.58
-    oav.parameters.micronsPerYPixel = 1.58
-    oav.parameters.beam_centre_i = 517
-    oav.parameters.beam_centre_j = 350
+    set_mock_value(oav.grid_snapshot.x_size, 1024)
+    set_mock_value(oav.grid_snapshot.y_size, 768)
 
     oav.snapshot.trigger = MagicMock(return_value=NullStatus())
-    oav.zoom_controller.zrst.set("1.0x")
-    oav.zoom_controller.onst.set("2.0x")
-    oav.zoom_controller.twst.set("3.0x")
-    oav.zoom_controller.thst.set("5.0x")
-    oav.zoom_controller.frst.set("7.0x")
-    oav.zoom_controller.fvst.set("9.0x")
-    oav.proc.port_name.sim_put("proc")  # type: ignore
-    oav.cam.port_name.sim_put("CAM")  # type: ignore
     oav.grid_snapshot.trigger = MagicMock(return_value=NullStatus())
     return oav
 
@@ -640,8 +625,6 @@ def fake_create_rotation_devices(
     xbpm_feedback: XBPMFeedback,
 ):
     set_mock_value(smargon.omega.max_velocity, 131)
-    oav.zoom_controller.zrst.sim_put("1.0x")  # type: ignore
-    oav.zoom_controller.fvst.sim_put("5.0x")  # type: ignore
 
     return RotationScanComposite(
         attenuator=attenuator,