diff --git a/.vscode/settings.json b/.vscode/settings.json index 4e927f6bc..744bda41b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,5 @@ { - "python.testing.pytestArgs": [], + "python.testing.pytestArgs": ["-v"], "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true, "python.analysis.autoImportCompletions": true, diff --git a/docs/param_hierarchy.puml b/docs/param_hierarchy.puml index 9646ae6e8..9cf7e2fd2 100644 --- a/docs/param_hierarchy.puml +++ b/docs/param_hierarchy.puml @@ -1,4 +1,4 @@ -@startuml +@startuml hyperion_parameter_model 'https://plantuml.com/class-diagram title Hyperion Parameter Model @@ -13,11 +13,10 @@ package Mixins { class XyzStarts class OptionalGonioAngleStarts class SplitScan + class RotationScanPerSweep + class RotationExperiment } -class HyperionParameters -note bottom: Base class for all experiment parameter models - package Experiments { class DiffractionExperiment class DiffractionExperimentWithSample @@ -25,10 +24,15 @@ package Experiments { class GridScanWithEdgeDetect class PinTipCentreThenXrayCentre class RotationScan + class MultiRotationScan class RobotLoadThenCentre class SpecifiedGridScan class ThreeDGridScan } + +class HyperionParameters +note bottom: Base class for all experiment parameter models + class TemporaryIspybExtras note bottom: To be removed @@ -45,6 +49,10 @@ BaseModel <|-- WithScan BaseModel <|-- XyzStarts RotationScan *-- TemporaryIspybExtras +MultiRotationScan *-- TemporaryIspybExtras +OptionalGonioAngleStarts <|-- RotationScanPerSweep +OptionalXyzStarts <|-- RotationScanPerSweep +DiffractionExperimentWithSample <|-- RotationExperiment HyperionParameters <|-- DiffractionExperiment WithSnapshot <|-- DiffractionExperiment DiffractionExperiment <|-- DiffractionExperimentWithSample @@ -58,8 +66,12 @@ WithScan <|-- SpecifiedGridScan SpecifiedGridScan <|-- ThreeDGridScan SplitScan <|-- ThreeDGridScan WithOavCentring <|-- GridCommon -DiffractionExperimentWithSample <|-- RotationScan -OptionalXyzStarts <|-- RotationScan +WithScan <|-- RotationScan +RotationScanPerSweep <|-- RotationScan +MultiRotationScan *-- RotationScanPerSweep +RotationExperiment <|-- RotationScan +RotationExperiment <|-- MultiRotationScan +SplitScan <|-- MultiRotationScan XyzStarts <|-- SpecifiedGridScan OptionalGonioAngleStarts <|-- GridCommon OptionalGonioAngleStarts <|-- RotationScan diff --git a/setup.cfg b/setup.cfg index 4d1bd2782..4c140471c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,18 +19,19 @@ install_requires = # If a constraint is not set here or if the dependency is pinned to a hash # it will be auto-pinned to the latest release version by the pre-release workflow # + annotated_types flask-restful + ispyb + nexgen + numpy opentelemetry-distro opentelemetry-exporter-jaeger - semver pydantic - scipy + pyepics pyzmq scanspec - numpy - pyepics - ispyb - nexgen + scipy + semver # # These dependencies may be issued as pre-release versions and should have a pin constraint # as by default pip-install will not upgrade to a pre-release. diff --git a/src/hyperion/device_setup_plans/manipulate_sample.py b/src/hyperion/device_setup_plans/manipulate_sample.py index d0149ac9d..ae2af0503 100644 --- a/src/hyperion/device_setup_plans/manipulate_sample.py +++ b/src/hyperion/device_setup_plans/manipulate_sample.py @@ -82,22 +82,22 @@ def cleanup_sample_environment( def move_x_y_z( smargon: Smargon, - x: float | None = None, - y: float | None = None, - z: float | None = None, + x_mm: float | None = None, + y_mm: float | None = None, + z_mm: float | None = None, wait=False, group="move_x_y_z", ): """Move the x, y, and z axes of the given smargon to the specified position. All axes are optional.""" - LOGGER.info(f"Moving smargon to x, y, z: {(x, y, z)}") - if x: - yield from bps.abs_set(smargon.x, x, group=group) - if y: - yield from bps.abs_set(smargon.y, y, group=group) - if z: - yield from bps.abs_set(smargon.z, z, group=group) + LOGGER.info(f"Moving smargon to x, y, z: {(x_mm, y_mm, z_mm)}") + if x_mm: + yield from bps.abs_set(smargon.x, x_mm, group=group) + if y_mm: + yield from bps.abs_set(smargon.y, y_mm, group=group) + if z_mm: + yield from bps.abs_set(smargon.z, z_mm, group=group) if wait: yield from bps.wait(group) diff --git a/src/hyperion/device_setup_plans/position_detector.py b/src/hyperion/device_setup_plans/position_detector.py index d40482617..1db189d0a 100644 --- a/src/hyperion/device_setup_plans/position_detector.py +++ b/src/hyperion/device_setup_plans/position_detector.py @@ -13,4 +13,4 @@ def set_detector_z_position( def set_shutter(detector_motion: DetectorMotion, state: ShutterState, group=None): LOGGER.info(f"Setting shutter to {state} ({group})") - yield from bps.abs_set(detector_motion.shutter, int(state), group=group) + yield from bps.abs_set(detector_motion.shutter, state, group=group) diff --git a/src/hyperion/device_setup_plans/setup_zebra.py b/src/hyperion/device_setup_plans/setup_zebra.py index 96ec5f519..ad7069159 100644 --- a/src/hyperion/device_setup_plans/setup_zebra.py +++ b/src/hyperion/device_setup_plans/setup_zebra.py @@ -98,8 +98,7 @@ def setup_zebra_for_rotation( "Disallowed rotation direction provided to Zebra setup plan. " "Use RotationDirection.POSITIVE or RotationDirection.NEGATIVE." ) - # TODO Actually set the rotation direction in here. - # See https://github.com/DiamondLightSource/hyperion/issues/1273 + yield from bps.abs_set(zebra.pc.dir, direction.value, group=group) LOGGER.info("ZEBRA SETUP: START") # must be on for shutter trigger to be enabled yield from bps.abs_set(zebra.inputs.soft_in_1, SoftInState.YES, group=group) diff --git a/src/hyperion/experiment_plans/__init__.py b/src/hyperion/experiment_plans/__init__.py index 5b89f8bfd..54b87452d 100644 --- a/src/hyperion/experiment_plans/__init__.py +++ b/src/hyperion/experiment_plans/__init__.py @@ -13,12 +13,16 @@ from hyperion.experiment_plans.robot_load_then_centre_plan import ( robot_load_then_centre, ) -from hyperion.experiment_plans.rotation_scan_plan import rotation_scan +from hyperion.experiment_plans.rotation_scan_plan import ( + multi_rotation_scan, + rotation_scan, +) __all__ = [ "flyscan_xray_centre", "grid_detect_then_xray_centre", "rotation_scan", "pin_tip_centre_then_xray_centre", + "multi_rotation_scan", "robot_load_then_centre", ] diff --git a/src/hyperion/experiment_plans/experiment_registry.py b/src/hyperion/experiment_plans/experiment_registry.py index 39252c886..5b4fd8c34 100644 --- a/src/hyperion/experiment_plans/experiment_registry.py +++ b/src/hyperion/experiment_plans/experiment_registry.py @@ -21,7 +21,7 @@ RobotLoadThenCentre, ThreeDGridScan, ) -from hyperion.parameters.rotation import RotationScan +from hyperion.parameters.rotation import MultiRotationScan, RotationScan def not_implemented(): @@ -38,6 +38,7 @@ class ExperimentRegistryEntry(TypedDict): ThreeDGridScan | GridScanWithEdgeDetect | RotationScan + | MultiRotationScan | PinTipCentreThenXrayCentre | RobotLoadThenCentre ] @@ -70,6 +71,11 @@ class ExperimentRegistryEntry(TypedDict): "param_type": RobotLoadThenCentre, "callbacks_factory": create_robot_load_and_centre_callbacks, }, + "multi_rotation_scan": { + "setup": rotation_scan_plan.create_devices, + "param_type": MultiRotationScan, + "callbacks_factory": create_rotation_callbacks, + }, } diff --git a/src/hyperion/experiment_plans/flyscan_xray_centre_plan.py b/src/hyperion/experiment_plans/flyscan_xray_centre_plan.py index 5250049de..547847143 100755 --- a/src/hyperion/experiment_plans/flyscan_xray_centre_plan.py +++ b/src/hyperion/experiment_plans/flyscan_xray_centre_plan.py @@ -147,6 +147,7 @@ def flyscan_xray_centre( md={ "subplan_name": CONST.PLAN.GRIDSCAN_OUTER, CONST.TRIGGER.ZOCALO: CONST.PLAN.DO_FGS, + "zocalo_environment": parameters.zocalo_environment, "hyperion_parameters": parameters.json(), "activate_callbacks": [ "GridscanNexusFileCallback", @@ -291,7 +292,6 @@ def run_gridscan( feature_controlled.fgs_motors, fgs_composite.eiger, fgs_composite.synchrotron, - parameters.zocalo_environment, [parameters.scan_points_first_grid, parameters.scan_points_second_grid], parameters.scan_indices, do_during_run=read_during_collection, @@ -303,7 +303,6 @@ def kickoff_and_complete_gridscan( gridscan: FastGridScanCommon, eiger: EigerDetector, synchrotron: Synchrotron, - zocalo_environment: str, scan_points: list[AxesPoints[Axis]], scan_start_indices: list[int], do_during_run: Callable[[], MsgGenerator] | None = None, @@ -313,7 +312,6 @@ def kickoff_and_complete_gridscan( @bpp.run_decorator( md={ "subplan_name": CONST.PLAN.DO_FGS, - "zocalo_environment": zocalo_environment, "scan_points": scan_points, "scan_start_indices": scan_start_indices, } diff --git a/src/hyperion/experiment_plans/rotation_scan_plan.py b/src/hyperion/experiment_plans/rotation_scan_plan.py index ea7cd504f..a8937bb92 100644 --- a/src/hyperion/experiment_plans/rotation_scan_plan.py +++ b/src/hyperion/experiment_plans/rotation_scan_plan.py @@ -47,7 +47,10 @@ ) from hyperion.log import LOGGER from hyperion.parameters.constants import CONST -from hyperion.parameters.rotation import RotationScan +from hyperion.parameters.rotation import ( + MultiRotationScan, + RotationScan, +) from hyperion.utils.aperturescatterguard import ( load_default_aperture_scatterguard_positions_if_unset, ) @@ -91,11 +94,6 @@ def create_devices(context: BlueskyContext) -> RotationScanComposite: # Use a slightly larger time to acceleration than EPICS as it's better to be cautious ACCELERATION_MARGIN = 1.5 -ROTATION_DIRECTION = { - RotationDirection.POSITIVE: 1, - RotationDirection.NEGATIVE: -1, -} - @dataclasses.dataclass class RotationMotionProfile: @@ -125,7 +123,7 @@ def calculate_motion_profile( See https://github.com/DiamondLightSource/hyperion/wiki/rotation-scan-geometry for a simple pictorial explanation.""" - direction = ROTATION_DIRECTION[params.rotation_direction] + direction = params.rotation_direction.multiplier num_images = params.num_images shutter_time_s = params.shutter_opening_time_s image_width_deg = params.rotation_increment_deg @@ -137,20 +135,44 @@ def calculate_motion_profile( LOGGER.info( f"{num_images=}, {shutter_time_s=}, {image_width_deg=}, {exposure_time_s=}, {direction=}" ) - LOGGER.info(f"{(scan_width_deg := num_images * params.rotation_increment_deg)=}") - LOGGER.info(f"{(speed_for_rotation_deg_s := image_width_deg / exposure_time_s)=}") + + scan_width_deg = num_images * params.rotation_increment_deg + LOGGER.info(f"{scan_width_deg=} = {num_images=} * {params.rotation_increment_deg=}") + + speed_for_rotation_deg_s = image_width_deg / exposure_time_s + LOGGER.info("speed_for_rotation_deg_s = image_width_deg / exposure_time_s") LOGGER.info( - f"{(acceleration_offset_deg := motor_time_to_speed_s * speed_for_rotation_deg_s)=}" + f"{speed_for_rotation_deg_s=} = {image_width_deg=} / {exposure_time_s=}" ) + + acceleration_offset_deg = motor_time_to_speed_s * speed_for_rotation_deg_s LOGGER.info( - f"{(start_motion_deg := start_scan_deg - (acceleration_offset_deg * direction))=}" + f"{acceleration_offset_deg=} = {motor_time_to_speed_s=} * {speed_for_rotation_deg_s=}" ) + + start_motion_deg = start_scan_deg - (acceleration_offset_deg * direction) LOGGER.info( - f"{(shutter_opening_deg := speed_for_rotation_deg_s * shutter_time_s)=}" + f"{start_motion_deg=} = {start_scan_deg=} - ({acceleration_offset_deg=} * {direction=})" ) - LOGGER.info(f"{(total_exposure_s := num_images * exposure_time_s)=}") + + shutter_opening_deg = speed_for_rotation_deg_s * shutter_time_s LOGGER.info( - f"{(distance_to_move_deg := (scan_width_deg + shutter_opening_deg + acceleration_offset_deg * 2) * direction)=}" + f"{shutter_opening_deg=} = {speed_for_rotation_deg_s=} * {shutter_time_s=}" + ) + + shutter_opening_deg = speed_for_rotation_deg_s * shutter_time_s + LOGGER.info( + f"{shutter_opening_deg=} = {speed_for_rotation_deg_s=} * {shutter_time_s=}" + ) + + total_exposure_s = num_images * exposure_time_s + LOGGER.info(f"{total_exposure_s=} = {num_images=} * {exposure_time_s=}") + + distance_to_move_deg = ( + scan_width_deg + shutter_opening_deg + acceleration_offset_deg * 2 + ) * direction + LOGGER.info( + f"{distance_to_move_deg=} = ({scan_width_deg=} + {shutter_opening_deg=} + {acceleration_offset_deg=} * 2) * {direction=})" ) return RotationMotionProfile( @@ -173,15 +195,14 @@ def rotation_scan_plan( params: RotationScan, motion_values: RotationMotionProfile, ): - """A plan to collect diffraction images from a sample continuously rotating about - a fixed axis - for now this axis is limited to omega. Only does the scan itself, no - setup tasks.""" + """A stub plan to collect diffraction images from a sample continuously rotating + about a fixed axis - for now this axis is limited to omega. + Needs additional setup of the sample environment and a wrapper to clean up.""" @bpp.set_run_key_decorator(CONST.PLAN.ROTATION_MAIN) @bpp.run_decorator( md={ "subplan_name": CONST.PLAN.ROTATION_MAIN, - "zocalo_environment": params.zocalo_environment, "scan_points": [params.scan_points], } ) @@ -222,6 +243,7 @@ def _rotation_scan_plan( LOGGER.info("Wait for any previous moves...") # wait for all the setup tasks at once + yield from bps.wait(CONST.WAIT.MOVE_GONIO_TO_START) yield from bps.wait("setup_senv") yield from bps.wait("move_to_rotation_start") @@ -265,14 +287,54 @@ def _rotation_scan_plan( yield from _rotation_scan_plan(motion_values, composite) -def cleanup_plan(composite: RotationScanComposite, max_vel: float, **kwargs): +def _cleanup_plan(composite: RotationScanComposite, **kwargs): LOGGER.info("Cleaning up after rotation scan") + max_vel = yield from bps.rd(composite.smargon.omega.max_velocity) yield from cleanup_sample_environment(composite.detector_motion, group="cleanup") yield from bps.abs_set(composite.smargon.omega.velocity, max_vel, group="cleanup") yield from make_trigger_safe(composite.zebra, group="cleanup") yield from bpp.finalize_wrapper(disarm_zebra(composite.zebra), bps.wait("cleanup")) +def _move_and_rotation( + composite: RotationScanComposite, + params: RotationScan, + oav_params: OAVParameters, +): + motor_time_to_speed = yield from bps.rd(composite.smargon.omega.acceleration_time) + max_vel = yield from bps.rd(composite.smargon.omega.max_velocity) + motion_values = calculate_motion_profile(params, motor_time_to_speed, max_vel) + + def _div_by_1000_if_not_none(num: float | None): + return num / 1000 if num else num + + LOGGER.info("moving to position (if specified)") + yield from move_x_y_z( + composite.smargon, + _div_by_1000_if_not_none(params.x_start_um), + _div_by_1000_if_not_none(params.y_start_um), + _div_by_1000_if_not_none(params.z_start_um), + group=CONST.WAIT.MOVE_GONIO_TO_START, + ) + yield from move_phi_chi_omega( + composite.smargon, + params.phi_start_deg, + params.chi_start_deg, + group=CONST.WAIT.MOVE_GONIO_TO_START, + ) + if params.take_snapshots: + yield from bps.wait(CONST.WAIT.MOVE_GONIO_TO_START) + yield from setup_oav_snapshot_plan( + composite, params, motion_values.max_velocity_deg_s + ) + yield from oav_snapshot_plan(composite, params, oav_params) + yield from rotation_scan_plan( + composite, + params, + motion_values, + ) + + def rotation_scan( composite: RotationScanComposite, parameters: RotationScan, @@ -296,24 +358,11 @@ def rotation_scan( def rotation_scan_plan_with_stage_and_cleanup( params: RotationScan, ): - motor_time_to_speed = yield from bps.rd( - composite.smargon.omega.acceleration_time - ) - max_vel = ( - yield from bps.rd(composite.smargon.omega.max_velocity) - or DEFAULT_MAX_VELOCITY - ) - motion_values = calculate_motion_profile( - params, - motor_time_to_speed, - max_vel, - ) - eiger: EigerDetector = composite.eiger eiger.set_detector_parameters(params.detector_params) @bpp.stage_decorator([eiger]) - @bpp.finalize_decorator(lambda: cleanup_plan(composite, max_vel)) + @bpp.finalize_decorator(lambda: _cleanup_plan(composite)) def rotation_with_cleanup_and_stage(params: RotationScan): assert composite.aperture_scatterguard.aperture_positions is not None LOGGER.info("setting up sample environment...") @@ -323,31 +372,64 @@ def rotation_with_cleanup_and_stage(params: RotationScan): params.transmission_frac, params.detector_params.detector_distance, ) - LOGGER.info("moving to position (if specified)") - yield from move_x_y_z( - composite.smargon, - params.x_start_um, - params.y_start_um, - params.z_start_um, - group="move_gonio_to_start", - ) - yield from move_phi_chi_omega( - composite.smargon, - params.phi_start_deg, - params.chi_start_deg, - group="move_gonio_to_start", - ) - yield from bps.wait("move_gonio_to_start") - if params.take_snapshots: - yield from setup_oav_snapshot_plan( - composite, params, motion_values.max_velocity_deg_s - ) - yield from oav_snapshot_plan(composite, params, oav_params) - - yield from rotation_scan_plan(composite, params, motion_values) + yield from _move_and_rotation(composite, params, oav_params) LOGGER.info("setting up and staging eiger...") yield from rotation_with_cleanup_and_stage(params) yield from rotation_scan_plan_with_stage_and_cleanup(parameters) + + +def multi_rotation_scan( + composite: RotationScanComposite, + parameters: MultiRotationScan, + oav_params: OAVParameters | None = None, +) -> MsgGenerator: + if not oav_params: + oav_params = OAVParameters(context="xrayCentring") + eiger: EigerDetector = composite.eiger + eiger.set_detector_parameters(parameters.detector_params) + assert composite.aperture_scatterguard.aperture_positions is not None + LOGGER.info("setting up sample environment...") + yield from begin_sample_environment_setup( + composite.detector_motion, + composite.attenuator, + parameters.transmission_frac, + parameters.detector_params.detector_distance, + ) + + @bpp.set_run_key_decorator("multi_rotation_scan") + @bpp.run_decorator( + md={ + "subplan_name": CONST.PLAN.ROTATION_MULTI, + "full_num_of_images": parameters.num_images, + "meta_data_run_number": parameters.detector_params.run_number, + "activate_callbacks": [ + "RotationISPyBCallback", + "RotationNexusFileCallback", + ], + } + ) + @bpp.stage_decorator([eiger]) + @bpp.finalize_decorator(lambda: _cleanup_plan(composite)) + def _multi_rotation_scan(): + for single_scan in parameters.single_rotation_scans: + + @bpp.set_run_key_decorator("rotation_scan") + @bpp.run_decorator( # attach experiment metadata to the start document + md={ + "subplan_name": CONST.PLAN.ROTATION_OUTER, + CONST.TRIGGER.ZOCALO: CONST.PLAN.ROTATION_MAIN, + "hyperion_parameters": single_scan.json(), + } + ) + def rotation_scan_core( + params: RotationScan, + ): + yield from _move_and_rotation(composite, params, oav_params) + + yield from rotation_scan_core(single_scan) + + LOGGER.info("setting up and staging eiger...") + yield from _multi_rotation_scan() diff --git a/src/hyperion/external_interaction/callbacks/rotation/nexus_callback.py b/src/hyperion/external_interaction/callbacks/rotation/nexus_callback.py index eb30166d1..742017dc4 100644 --- a/src/hyperion/external_interaction/callbacks/rotation/nexus_callback.py +++ b/src/hyperion/external_interaction/callbacks/rotation/nexus_callback.py @@ -38,6 +38,9 @@ def __init__(self) -> None: self.run_uid: str | None = None self.writer: NexusWriter | None = None self.descriptors: Dict[str, EventDescriptor] = {} + # used when multiple collections are made in one detector arming event: + self.full_num_of_images: int | None = None + self.meta_data_run_number: int | None = None def activity_gated_descriptor(self, doc: EventDescriptor): self.descriptors[doc["uid"]] = doc @@ -66,10 +69,13 @@ def activity_gated_event(self, doc: Event): ) vds_data_type = vds_type_based_on_bit_depth(doc["data"]["eiger_bit_depth"]) self.writer.create_nexus_file(vds_data_type) - NEXUS_LOGGER.info(f"Nexus file created at {self.writer.full_filename}") + NEXUS_LOGGER.info(f"Nexus file created at {self.writer.data_filename}") return doc def activity_gated_start(self, doc: RunStart): + if doc.get("subplan_name") == CONST.PLAN.ROTATION_MULTI: + self.full_num_of_images = doc.get("full_num_of_images") + self.meta_data_run_number = doc.get("meta_data_run_number") if doc.get("subplan_name") == CONST.PLAN.ROTATION_OUTER: self.run_uid = doc.get("uid") json_params = doc.get("hyperion_parameters") @@ -88,5 +94,9 @@ def activity_gated_start(self, doc: RunStart): parameters.scan_points, omega_start_deg=parameters.omega_start_deg, chi_start_deg=parameters.chi_start_deg or 0, - phi_start_deg=parameters.chi_start_deg or 0, + phi_start_deg=parameters.phi_start_deg or 0, + vds_start_index=parameters.nexus_vds_start_img, + full_num_of_images=self.full_num_of_images, + meta_data_run_number=self.meta_data_run_number, + rotation_direction=parameters.rotation_direction, ) diff --git a/src/hyperion/external_interaction/callbacks/xray_centre/nexus_callback.py b/src/hyperion/external_interaction/callbacks/xray_centre/nexus_callback.py index f4546d1de..772402d0c 100644 --- a/src/hyperion/external_interaction/callbacks/xray_centre/nexus_callback.py +++ b/src/hyperion/external_interaction/callbacks/xray_centre/nexus_callback.py @@ -90,6 +90,6 @@ def activity_gated_event(self, doc: Event) -> Event | None: doc["data"]["eiger_bit_depth"] ) nexus_writer.create_nexus_file(vds_data_type) - NEXUS_LOGGER.info(f"Nexus file created at {nexus_writer.full_filename}") + NEXUS_LOGGER.info(f"Nexus file created at {nexus_writer.data_filename}") return super().activity_gated_event(doc) diff --git a/src/hyperion/external_interaction/callbacks/zocalo_callback.py b/src/hyperion/external_interaction/callbacks/zocalo_callback.py index 26cf61f7a..3a633cb20 100644 --- a/src/hyperion/external_interaction/callbacks/zocalo_callback.py +++ b/src/hyperion/external_interaction/callbacks/zocalo_callback.py @@ -42,10 +42,11 @@ def start(self, doc: RunStart): ISPYB_LOGGER.info("Zocalo handler received start document.") if triggering_plan := doc.get(CONST.TRIGGER.ZOCALO): self.triggering_plan = triggering_plan - if self.triggering_plan and doc.get("subplan_name") == self.triggering_plan: assert isinstance(zocalo_environment := doc.get("zocalo_environment"), str) ISPYB_LOGGER.info(f"Zocalo environment set to {zocalo_environment}.") self.zocalo_interactor = ZocaloTrigger(zocalo_environment) + + if self.triggering_plan and doc.get("subplan_name") == self.triggering_plan: self.run_uid = doc.get("uid") assert isinstance(scan_points := doc.get("scan_points"), list) if ( diff --git a/src/hyperion/external_interaction/nexus/nexus_utils.py b/src/hyperion/external_interaction/nexus/nexus_utils.py index 8c58ccc07..dfce541b4 100644 --- a/src/hyperion/external_interaction/nexus/nexus_utils.py +++ b/src/hyperion/external_interaction/nexus/nexus_utils.py @@ -5,6 +5,7 @@ import numpy as np from dodal.devices.detector import DetectorParams +from dodal.devices.zebra import RotationDirection from nexgen.nxs_utils import Attenuator, Axis, Beam, Detector, EigerDetector, Goniometer from nexgen.nxs_utils.axes import TransformationType from numpy.typing import DTypeLike @@ -34,6 +35,7 @@ def create_goniometer_axes( x_y_z_increments: tuple[float, float, float] = (0.0, 0.0, 0.0), chi: float = 0.0, phi: float = 0.0, + rotation_direction: RotationDirection = RotationDirection.NEGATIVE, ): """Returns a Nexgen 'Goniometer' object with the dependency chain of I03's Smargon goniometer. If scan points is provided these values will be used in preference to @@ -50,7 +52,13 @@ def create_goniometer_axes( is provided. """ gonio_axes = [ - Axis("omega", ".", TransformationType.ROTATION, (-1.0, 0.0, 0.0), omega_start), + Axis( + "omega", + ".", + TransformationType.ROTATION, + (1.0 * rotation_direction.multiplier, 0.0, 0.0), + omega_start, + ), Axis( name="sam_z", depends="omega", @@ -118,7 +126,9 @@ def create_detector_parameters(detector_params: DetectorParams) -> Detector: return Detector( eiger_params, detector_axes, - detector_params.get_beam_position_pixels(detector_params.detector_distance), + list( + detector_params.get_beam_position_pixels(detector_params.detector_distance) + ), detector_params.exposure_time, [(-1.0, 0.0, 0.0), (0.0, -1.0, 0.0)], ) diff --git a/src/hyperion/external_interaction/nexus/write_nexus.py b/src/hyperion/external_interaction/nexus/write_nexus.py index 000670a3f..9773954ab 100644 --- a/src/hyperion/external_interaction/nexus/write_nexus.py +++ b/src/hyperion/external_interaction/nexus/write_nexus.py @@ -9,6 +9,7 @@ from pathlib import Path from typing import Optional +from dodal.devices.zebra import RotationDirection from dodal.utils import get_beamline_name from nexgen.nxs_utils import Attenuator, Beam, Detector, Goniometer, Source from nexgen.nxs_write.nxmx_writer import NXmxFileWriter @@ -35,6 +36,11 @@ def __init__( chi_start_deg: float = 0, phi_start_deg: float = 0, vds_start_index: int = 0, + # override default values when there is more than one collection per + # detector arming event: + full_num_of_images: int | None = None, + meta_data_run_number: int | None = None, + rotation_direction: RotationDirection = RotationDirection.NEGATIVE, ) -> None: self.beam: Optional[Beam] = None self.attenuator: Optional[Attenuator] = None @@ -46,21 +52,25 @@ def __init__( self.detector: Detector = create_detector_parameters(parameters.detector_params) self.source: Source = Source(get_beamline_name("S03")) self.directory: Path = Path(parameters.storage_directory) - self.filename: str = parameters.file_name self.start_index: int = vds_start_index - self.full_num_of_images: int = parameters.num_images - self.full_filename: str = parameters.detector_params.full_filename + self.full_num_of_images: int = full_num_of_images or parameters.num_images + self.data_filename: str = ( + f"{parameters.file_name}_{meta_data_run_number}" + if meta_data_run_number + else parameters.detector_params.full_filename + ) self.nexus_file: Path = ( - self.directory / f"{self.filename}_{self.run_number}.nxs" + self.directory / f"{parameters.file_name}_{self.run_number}.nxs" ) self.master_file: Path = ( - self.directory / f"{self.filename}_{self.run_number}_master.h5" + self.directory / f"{parameters.file_name}_{self.run_number}_master.h5" ) self.goniometer: Goniometer = create_goniometer_axes( omega_start_deg, self.scan_points, chi=chi_start_deg, phi=phi_start_deg, + rotation_direction=rotation_direction, ) def create_nexus_file(self, bit_depth: DTypeLike): @@ -88,7 +98,7 @@ def create_nexus_file(self, bit_depth: DTypeLike): self.full_num_of_images, ) NXmx_Writer.write( - image_filename=f"{self.full_filename}", + image_filename=f"{self.data_filename}", start_time=start_time, est_end_time=est_end_time, ) @@ -98,7 +108,7 @@ def create_nexus_file(self, bit_depth: DTypeLike): def get_image_datafiles(self, max_images_per_file=1000): return [ - self.directory / f"{self.full_filename}_{h5_num + 1:06}.h5" + self.directory / f"{self.data_filename}_{h5_num + 1:06}.h5" for h5_num in range( math.ceil(self.full_num_of_images / max_images_per_file) ) diff --git a/src/hyperion/parameters/components.py b/src/hyperion/parameters/components.py index 3e090913a..e5b2d9503 100644 --- a/src/hyperion/parameters/components.py +++ b/src/hyperion/parameters/components.py @@ -169,6 +169,7 @@ class DiffractionExperiment(HyperionParameters, WithSnapshot): demand_energy_ev: float | None = Field(default=None, gt=0) run_number: int | None = Field(default=None, ge=0) selected_aperture: AperturePositionGDANames | None = Field(default=None) + transmission_frac: float = Field(default=0.1) ispyb_experiment_type: IspybExperimentType storage_directory: str diff --git a/src/hyperion/parameters/constants.py b/src/hyperion/parameters/constants.py index f029aaea7..98539b5c3 100644 --- a/src/hyperion/parameters/constants.py +++ b/src/hyperion/parameters/constants.py @@ -30,6 +30,7 @@ class PlanNameConstants: GRIDSCAN_MAIN = "run_gridscan" DO_FGS = "do_fgs" # Rotation scan + ROTATION_MULTI = "multi_rotation_wrapper" ROTATION_OUTER = "rotation_scan_with_cleanup" ROTATION_MAIN = "rotation_scan_main" @@ -39,6 +40,7 @@ class PlanGroupCheckpointConstants: # For places to synchronise / stop and wait in plans, use as bluesky group names # Gridscan GRID_READY_FOR_DC = "ready_for_data_collection" + MOVE_GONIO_TO_START = "move_gonio_to_start" @dataclass(frozen=True) diff --git a/src/hyperion/parameters/gridscan.py b/src/hyperion/parameters/gridscan.py index 84840dcf1..778b01c90 100644 --- a/src/hyperion/parameters/gridscan.py +++ b/src/hyperion/parameters/gridscan.py @@ -36,7 +36,6 @@ class GridCommon( grid_width_um: float = Field(default=CONST.PARAM.GRIDSCAN.WIDTH_UM) exposure_time_s: float = Field(default=CONST.PARAM.GRIDSCAN.EXPOSURE_TIME_S) use_roi_mode: bool = Field(default=CONST.PARAM.GRIDSCAN.USE_ROI) - transmission_frac: float = Field(default=1) panda_runup_distance_mm: float = Field( default=CONST.HARDWARE.PANDA_FGS_RUN_UP_DEFAULT ) diff --git a/src/hyperion/parameters/rotation.py b/src/hyperion/parameters/rotation.py index ecad11d03..ce59b2988 100644 --- a/src/hyperion/parameters/rotation.py +++ b/src/hyperion/parameters/rotation.py @@ -1,7 +1,11 @@ from __future__ import annotations import os +from collections.abc import Iterator +from itertools import accumulate +from typing import Annotated +from annotated_types import Len from dodal.devices.detector import DetectorParams from dodal.devices.detector.det_dist_to_beam_converter import ( DetectorDistanceToBeamXYConverter, @@ -9,7 +13,7 @@ from dodal.devices.zebra import ( RotationDirection, ) -from pydantic import Field +from pydantic import Field, root_validator from scanspec.core import AxesPoints from scanspec.core import Path as ScanPath from scanspec.specs import Line @@ -21,32 +25,30 @@ OptionalGonioAngleStarts, OptionalXyzStarts, RotationAxis, + SplitScan, TemporaryIspybExtras, WithScan, ) from hyperion.parameters.constants import CONST, I03Constants -class RotationScan( - DiffractionExperimentWithSample, - WithScan, - OptionalGonioAngleStarts, - OptionalXyzStarts, -): +class RotationScanPerSweep(OptionalGonioAngleStarts, OptionalXyzStarts): omega_start_deg: float = Field(default=0) # type: ignore rotation_axis: RotationAxis = Field(default=RotationAxis.OMEGA) - shutter_opening_time_s: float = Field(default=CONST.I03.SHUTTER_TIME_S) scan_width_deg: float = Field(default=360, gt=0) - rotation_increment_deg: float = Field(default=0.1, gt=0) rotation_direction: RotationDirection = Field(default=RotationDirection.NEGATIVE) + nexus_vds_start_img: int = Field(default=0, ge=0) + ispyb_extras: TemporaryIspybExtras | None + + +class RotationExperiment(DiffractionExperimentWithSample): + shutter_opening_time_s: float = Field(default=CONST.I03.SHUTTER_TIME_S) + rotation_increment_deg: float = Field(default=0.1, gt=0) ispyb_experiment_type: IspybExperimentType = Field( default=IspybExperimentType.ROTATION ) - transmission_frac: float - ispyb_extras: TemporaryIspybExtras | None - @property - def detector_params(self): + def _detector_params(self, omega_start_deg: float): self.det_dist_to_beam_converter_path = ( self.det_dist_to_beam_converter_path or CONST.PARAM.DETECTOR.BEAM_XY_LUT_PATH @@ -63,7 +65,7 @@ def detector_params(self): directory=self.storage_directory, prefix=self.file_name, detector_distance=self.detector_distance_mm, - omega_start=self.omega_start_deg, + omega_start=omega_start_deg, omega_increment=self.rotation_increment_deg, num_images_per_trigger=self.num_images, num_triggers=1, @@ -75,6 +77,8 @@ def detector_params(self): **optional_args, ) + +class RotationScan(WithScan, RotationScanPerSweep, RotationExperiment): @property def ispyb_params(self): # pyright: ignore return RotationIspybParams( @@ -86,15 +90,21 @@ def ispyb_params(self): # pyright: ignore else [] ), ispyb_experiment_type=self.ispyb_experiment_type, + position=None, ) + @property + def detector_params(self): + return self._detector_params(self.omega_start_deg) + @property def scan_points(self) -> AxesPoints: scan_spec = Line( axis="omega", start=self.omega_start_deg, stop=( - self.scan_width_deg + self.omega_start_deg - self.rotation_increment_deg + self.omega_start_deg + + (self.scan_width_deg - self.rotation_increment_deg) ), num=self.num_images, ) @@ -104,3 +114,51 @@ def scan_points(self) -> AxesPoints: @property def num_images(self) -> int: return int(self.scan_width_deg / self.rotation_increment_deg) + + +class MultiRotationScan(RotationExperiment, SplitScan): + rotation_scans: Annotated[list[RotationScanPerSweep], Len(min_length=1)] + + def _single_rotation_scan(self, scan: RotationScanPerSweep) -> RotationScan: + # self has everything from RotationExperiment + params = self.dict() + del params["rotation_scans"] + # provided `scan` has everything from RotationScanPerSweep + params.update(scan.dict()) + # together they have everything for RotationScan + return RotationScan(**params) + + @root_validator(pre=False) + def validate_snapshot_directory(cls, values): + start_img = 0 + for scan in values["rotation_scans"]: + scan.nexus_vds_start_img = start_img + start_img += scan.scan_width_deg / values["rotation_increment_deg"] + return values + + @property + def single_rotation_scans(self) -> Iterator[RotationScan]: + for scan in self.rotation_scans: + yield self._single_rotation_scan(scan) + + def _num_images_per_scan(self): + return [ + int(scan.scan_width_deg / self.rotation_increment_deg) + for scan in self.rotation_scans + ] + + @property + def num_images(self): + return sum(self._num_images_per_scan()) + + @property + def scan_indices(self): + return list(accumulate([0, *self._num_images_per_scan()])) + + @property + def detector_params(self): + return self._detector_params(self.rotation_scans[0].omega_start_deg) + + @property + def ispyb_params(self): # pyright: ignore + raise ValueError("Please get ispyb params from one of the individual scans") diff --git a/src/hyperion/utils/validation.py b/src/hyperion/utils/validation.py index 10181d5e0..2c158bf6a 100644 --- a/src/hyperion/utils/validation.py +++ b/src/hyperion/utils/validation.py @@ -98,7 +98,6 @@ def fake_create_rotation_devices(): ) set_mock_value(smargon.omega.max_velocity, 131) - set_mock_value(dcm.energy_in_kev.user_readback, 12700) oav.zoom_controller.fvst.sim_put("1.0x") diff --git a/tests/conftest.py b/tests/conftest.py index 053145e78..1011fed85 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -47,9 +47,7 @@ from dodal.devices.zebra import Zebra from dodal.log import LOGGER as dodal_logger from dodal.log import set_up_all_logging_handlers -from ophyd.epics_motor import EpicsMotor from ophyd.sim import NullStatus -from ophyd.status import Status from ophyd_async.core import Device, DeviceVector, callback_on_mock_put, set_mock_value from ophyd_async.core.async_status import AsyncStatus from ophyd_async.epics.motion.motor import Motor @@ -75,7 +73,7 @@ do_default_logging_setup, ) from hyperion.parameters.gridscan import GridScanWithEdgeDetect, ThreeDGridScan -from hyperion.parameters.rotation import RotationScan +from hyperion.parameters.rotation import MultiRotationScan, RotationScan i03.DAQ_CONFIGURATION_PATH = "tests/test_data/test_daq_configuration" @@ -178,20 +176,6 @@ def stop_event_loop(): del RE -def mock_set(motor: EpicsMotor, val): - motor.user_setpoint.sim_put(val) # type: ignore - motor.user_readback.sim_put(val) # type: ignore - return Status(done=True, success=True) - - -def patch_motor(motor: EpicsMotor): - return patch.object(motor, "set", MagicMock(side_effect=partial(mock_set, motor))) - - -async def mock_good_coroutine(): - return asyncio.sleep(0) - - def pass_on_mock(motor, call_log: MagicMock | None = None): def _pass_on_mock(value, **kwargs): set_mock_value(motor.user_readback, value) @@ -252,17 +236,25 @@ def test_rotation_params_nomove(): ) +@pytest.fixture +def test_multi_rotation_params(): + return MultiRotationScan( + **raw_params_from_file( + "tests/test_data/parameter_json_files/good_test_multi_rotation_scan_parameters.json" + ) + ) + + @pytest.fixture def done_status(): - s = Status() - s.set_finished() - return s + return NullStatus() @pytest.fixture def eiger(done_status): eiger = i03.eiger(fake_with_ophyd_sim=True) eiger.stage = MagicMock(return_value=done_status) + eiger.do_arm.set = MagicMock(return_value=done_status) eiger.unstage = MagicMock(return_value=done_status) return eiger @@ -297,7 +289,7 @@ def zebra(): def mock_side(*args, **kwargs): set_mock_value(zebra.pc.arm.armed, *args, **kwargs) - return Status(done=True, success=True) + return NullStatus() zebra.pc.arm.set = MagicMock(side_effect=mock_side) return zebra @@ -314,11 +306,9 @@ def fast_grid_scan(): @pytest.fixture -def detector_motion(): +def detector_motion(RE): det = i03.detector_motion(fake_with_ophyd_sim=True) - det.z.user_setpoint._use_limits = False - - with patch_motor(det.z): + with patch_async_motor(det.z): yield det @@ -333,8 +323,7 @@ def s4_slit_gaps(): @pytest.fixture -def synchrotron(): - RunEngine() # A RE is needed to start the bluesky loop +def synchrotron(RE): synchrotron = i03.synchrotron(fake_with_ophyd_sim=True) set_mock_value(synchrotron.synchrotron_mode, SynchrotronMode.USER) set_mock_value(synchrotron.top_up_start_countdown, 10) @@ -541,7 +530,7 @@ def fake_create_devices( detector_motion: DetectorMotion, aperture_scatterguard: ApertureScatterguard, ): - mock_omega_sets = MagicMock(return_value=Status(done=True, success=True)) + mock_omega_sets = MagicMock(return_value=NullStatus()) smargon.omega.velocity.set = mock_omega_sets smargon.omega.set = mock_omega_sets @@ -573,7 +562,6 @@ def fake_create_rotation_devices( dcm: DCM, robot: BartRobot, oav: OAV, - done_status, ): set_mock_value(smargon.omega.max_velocity, 131) oav.zoom_controller.onst.sim_put("1.0x") # type: ignore @@ -756,6 +744,113 @@ def sim_run_engine(): return RunEngineSimulator() +class DocumentCapturer: + """A utility which can be subscribed to the RunEngine in place of a callback in order + to intercept documents and make assertions about their contents""" + + def __init__(self) -> None: + self.docs_received: list[tuple[str, dict[str, Any]]] = [] + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + self.docs_received.append((args[0], args[1])) + + @staticmethod + def is_match( + doc: tuple[str, dict[str, Any]], + name: str, + has_fields: Sequence[str] = [], + matches_fields: dict[str, Any] = {}, + ): + """Returns True if the given document: + - has the same name + - contains all the fields in has_fields + - contains all the fields in matches_fields with the same content""" + + return ( + doc[0] == name + and all(f in doc[1].keys() for f in has_fields) + and matches_fields.items() <= doc[1].items() + ) + + @staticmethod + def get_matches( + docs: list[tuple[str, dict[str, Any]]], + name: str, + has_fields: Sequence[str] = [], + matches_fields: dict[str, Any] = {}, + ): + """Get all the docs from docs which: + - have the same name + - contain all the fields in has_fields + - contain all the fields in matches_fields with the same content""" + return list( + filter( + partial( + DocumentCapturer.is_match, + name=name, + has_fields=has_fields, + matches_fields=matches_fields, + ), + docs, + ) + ) + + @staticmethod + def assert_doc( + docs: list[tuple[str, dict[str, Any]]], + name: str, + has_fields: Sequence[str] = [], + matches_fields: dict[str, Any] = {}, + does_exist: bool = True, + ): + """Assert that a matching doc has been recieved by the sim, + and returns the first match if it is meant to exist""" + matches = DocumentCapturer.get_matches(docs, name, has_fields, matches_fields) + if does_exist: + assert matches + return matches[0] + else: + assert matches == [] + + @staticmethod + def get_docs_until( + docs: list[tuple[str, dict[str, Any]]], + name: str, + has_fields: Sequence[str] = [], + matches_fields: dict[str, Any] = {}, + ): + """return all the docs from the list of docs until the first matching one""" + for i, doc in enumerate(docs): + if DocumentCapturer.is_match(doc, name, has_fields, matches_fields): + return docs[: i + 1] + raise ValueError(f"Doc {name=}, {has_fields=}, {matches_fields=} not found") + + @staticmethod + def get_docs_from( + docs: list[tuple[str, dict[str, Any]]], + name: str, + has_fields: Sequence[str] = [], + matches_fields: dict[str, Any] = {}, + ): + """return all the docs from the list of docs after the first matching one""" + for i, doc in enumerate(docs): + if DocumentCapturer.is_match(doc, name, has_fields, matches_fields): + return docs[i:] + raise ValueError(f"Doc {name=}, {has_fields=}, {matches_fields=} not found") + + @staticmethod + def assert_events_and_data_in_order( + docs: list[tuple[str, dict[str, Any]]], + match_data_keys_list: Sequence[Sequence[str]], + ): + for event_data_keys in match_data_keys_list: + docs = DocumentCapturer.get_docs_from(docs, "event") + doc = docs.pop(0)[1]["data"] + assert all( + k in doc.keys() for k in event_data_keys + ), f"One of {event_data_keys=} not in {doc}" + + @pytest.fixture def feature_flags(): return FeatureFlags( diff --git a/tests/system_tests/experiment_plans/test_fgs_plan.py b/tests/system_tests/experiment_plans/test_fgs_plan.py index d1d41171e..cafdc7f40 100644 --- a/tests/system_tests/experiment_plans/test_fgs_plan.py +++ b/tests/system_tests/experiment_plans/test_fgs_plan.py @@ -14,7 +14,7 @@ ) from dodal.devices.aperturescatterguard import AperturePositions from dodal.devices.smargon import Smargon -from ophyd.status import Status +from ophyd.sim import NullStatus from ophyd_async.core import set_mock_value from hyperion.device_setup_plans.read_hardware_for_setup import ( @@ -116,8 +116,8 @@ async def fxc_composite(): ) composite.eiger.cam.manual_trigger.put("Yes") composite.eiger.odin.check_odin_initialised = lambda: (True, "") - composite.eiger.stage = MagicMock(return_value=Status(done=True, success=True)) - composite.eiger.unstage = MagicMock(return_value=Status(done=True, success=True)) + composite.eiger.stage = MagicMock(return_value=NullStatus()) + composite.eiger.unstage = MagicMock(return_value=NullStatus()) set_mock_value(composite.xbpm_feedback.pos_ok, True) set_mock_value(composite.xbpm_feedback.pos_stable, True) @@ -125,14 +125,12 @@ async def fxc_composite(): return composite -@pytest.mark.asyncio @pytest.mark.s03 def test_s03_devices_connect(fxc_composite: FlyScanXRayCentreComposite): assert fxc_composite.aperture_scatterguard assert fxc_composite.backlight -@pytest.mark.asyncio @pytest.mark.s03 def test_read_hardware_pre_collection( RE: RunEngine, @@ -160,7 +158,6 @@ def read_run(u, s, g, r, a, f, dcm, ap_sg, sm): ) -@pytest.mark.asyncio @pytest.mark.s03 def test_xbpm_feedback_decorator( RE: RunEngine, @@ -184,7 +181,6 @@ def decorated_plan(): assert fxc_composite.xbpm_feedback.pos_stable.get() == 1 -@pytest.mark.asyncio @pytest.mark.s03 @patch("bluesky.plan_stubs.wait", autospec=True) @patch("bluesky.plan_stubs.kickoff", autospec=True) @@ -220,7 +216,6 @@ def test_full_plan_tidies_at_end( set_shutter_to_manual.assert_called_once() -@pytest.mark.asyncio @pytest.mark.s03 @patch("bluesky.plan_stubs.wait", autospec=True) @patch("bluesky.plan_stubs.kickoff", autospec=True) @@ -250,7 +245,6 @@ def test_full_plan_tidies_at_end_when_plan_fails( @patch("hyperion.external_interaction.callbacks.zocalo_callback.ZocaloTrigger") -@pytest.mark.asyncio @pytest.mark.s03 def test_GIVEN_scan_invalid_WHEN_plan_run_THEN_ispyb_entry_made_but_no_zocalo_entry( zocalo_trigger: MagicMock, @@ -282,7 +276,6 @@ def test_GIVEN_scan_invalid_WHEN_plan_run_THEN_ispyb_entry_made_but_no_zocalo_en zocalo_trigger.run_start.assert_not_called() -@pytest.mark.asyncio @pytest.mark.s03 def test_complete_xray_centre_plan_with_no_callbacks_falls_back_to_centre( RE: RunEngine, @@ -317,7 +310,6 @@ def zocalo_trigger(): assert fxc_composite.sample_motors.z.user_readback.get() == pytest.approx(-1) -@pytest.mark.asyncio @pytest.mark.s03 def test_complete_xray_centre_plan_with_callbacks_moves_to_centre( RE: RunEngine, diff --git a/tests/system_tests/experiment_plans/test_plan_system.py b/tests/system_tests/experiment_plans/test_plan_system.py index 96cab8765..3b390519d 100644 --- a/tests/system_tests/experiment_plans/test_plan_system.py +++ b/tests/system_tests/experiment_plans/test_plan_system.py @@ -14,7 +14,6 @@ @pytest.mark.s03 -@pytest.mark.asyncio async def test_getting_data_for_ispyb(): undulator = Undulator( f"{CONST.SIM.INSERTION_PREFIX}-MO-SERVC-01:", name="undulator" diff --git a/tests/system_tests/external_interaction/callbacks/test_external_callbacks.py b/tests/system_tests/external_interaction/callbacks/test_external_callbacks.py index ad5a2ac5e..f3d44d36d 100644 --- a/tests/system_tests/external_interaction/callbacks/test_external_callbacks.py +++ b/tests/system_tests/external_interaction/callbacks/test_external_callbacks.py @@ -136,7 +136,6 @@ def plan(): RE(plan()) -@pytest.mark.asyncio @pytest.mark.s03 async def test_external_callbacks_handle_gridscan_ispyb_and_zocalo( RE_with_external_callbacks: RunEngine, diff --git a/tests/system_tests/external_interaction/test_zocalo_system.py b/tests/system_tests/external_interaction/test_zocalo_system.py index 3aee45a24..e4f2dcfbe 100644 --- a/tests/system_tests/external_interaction/test_zocalo_system.py +++ b/tests/system_tests/external_interaction/test_zocalo_system.py @@ -102,7 +102,6 @@ def inner_plan(): return inner -@pytest.mark.asyncio @pytest.mark.s03 async def test_given_a_result_with_no_diffraction_when_zocalo_called_then_move_to_fallback( run_zocalo_with_dev_ispyb, zocalo_env @@ -112,7 +111,6 @@ async def test_given_a_result_with_no_diffraction_when_zocalo_called_then_move_t assert np.allclose(centre, fallback) -@pytest.mark.asyncio @pytest.mark.s03 async def test_given_a_result_with_no_diffraction_ispyb_comment_updated( run_zocalo_with_dev_ispyb, zocalo_env, fetch_comment @@ -123,7 +121,6 @@ async def test_given_a_result_with_no_diffraction_ispyb_comment_updated( assert "Zocalo found no crystals in this gridscan." in comment -@pytest.mark.asyncio @pytest.mark.s03 async def test_zocalo_adds_nonzero_comment_time( run_zocalo_with_dev_ispyb, zocalo_env, fetch_comment @@ -138,7 +135,6 @@ async def test_zocalo_adds_nonzero_comment_time( assert time_s < 180 -@pytest.mark.asyncio @pytest.mark.s03 async def test_given_a_single_crystal_result_ispyb_comment_updated( run_zocalo_with_dev_ispyb, zocalo_env, fetch_comment @@ -150,7 +146,6 @@ async def test_given_a_single_crystal_result_ispyb_comment_updated( assert "Size (grid boxes)" in comment -@pytest.mark.asyncio @pytest.mark.s03 async def test_given_a_result_with_multiple_crystals_ispyb_comment_updated( run_zocalo_with_dev_ispyb, zocalo_env, fetch_comment diff --git a/tests/test_data/parameter_json_files/good_test_multi_rotation_scan_parameters.json b/tests/test_data/parameter_json_files/good_test_multi_rotation_scan_parameters.json new file mode 100644 index 000000000..62a27b933 --- /dev/null +++ b/tests/test_data/parameter_json_files/good_test_multi_rotation_scan_parameters.json @@ -0,0 +1,50 @@ +{ + "parameter_model_version": "5.0.0", + "comment": "test", + "det_dist_to_beam_converter_path": "tests/test_data/test_lookup_table.txt", + "storage_directory": "/tmp/dls/i03/data/2024/cm31105-4/auto/123456/", + "detector_distance_mm": 100.0, + "demand_energy_ev": 100, + "exposure_time_s": 0.1, + "insertion_prefix": "SR03S", + "file_name": "file_name", + "run_number": 0, + "sample_id": 123456, + "shutter_opening_time_s": 0.6, + "visit": "cm31105-4", + "zocalo_environment": "dev_artemis", + "transmission_frac": 0.1, + "rotation_increment_deg": 0.1, + "selected_aperture": "SMALL_APERTURE", + "rotation_scans": [{ + "rotation_axis": "omega", + "rotation_direction": "Negative", + "scan_width_deg": 180.0, + "omega_start_deg": 0.0, + "phi_start_deg": 0.47, + "chi_start_deg": 23.85, + "x_start_um": 1.0, + "y_start_um": 2.0, + "z_start_um": 3.0 + },{ + "rotation_axis": "omega", + "rotation_direction": "Negative", + "scan_width_deg": 90.0, + "omega_start_deg": 180.0, + "phi_start_deg": 0.47, + "chi_start_deg": 4.7, + "x_start_um": 3.0, + "y_start_um": 2.0, + "z_start_um": 1.0 + },{ + "rotation_axis": "omega", + "rotation_direction": "Positive", + "scan_width_deg": 360.0, + "omega_start_deg": 270.0, + "phi_start_deg": 0.47, + "chi_start_deg": 45, + "x_start_um": 6.0, + "y_start_um": 7.0, + "z_start_um": 8.0 + }] +} \ No newline at end of file diff --git a/tests/unit_tests/experiment_plans/conftest.py b/tests/unit_tests/experiment_plans/conftest.py index 93f78084a..e9a383bff 100644 --- a/tests/unit_tests/experiment_plans/conftest.py +++ b/tests/unit_tests/experiment_plans/conftest.py @@ -9,7 +9,7 @@ from dodal.devices.synchrotron import SynchrotronMode from dodal.devices.zocalo import ZocaloResults, ZocaloTrigger from event_model import Event -from ophyd.sim import make_fake_device +from ophyd_async.core import DeviceCollector from ophyd_async.core.async_status import AsyncStatus from hyperion.external_interaction.callbacks.common.callback_util import ( @@ -60,6 +60,24 @@ def make_event_doc(data, descriptor="abc123") -> Event: } +@pytest.fixture +def sim_run_engine_for_rotation(sim_run_engine): + sim_run_engine.add_handler( + "read", + lambda msg: {"values": {"value": SynchrotronMode.USER}}, + "synchrotron-synchrotron_mode", + ) + sim_run_engine.add_handler( + "read", + lambda msg: {"values": {"value": -1}}, + "synchrotron-top_up_start_countdown", + ) + sim_run_engine.add_handler( + "read", lambda msg: {"values": {"value": -1}}, "smargon_omega" + ) + return sim_run_engine + + def mock_zocalo_trigger(zocalo: ZocaloResults, result): @AsyncStatus.wrap async def mock_complete(results): @@ -163,13 +181,15 @@ def fake_read(obj, initial_positions, _): @pytest.fixture def simple_beamline(detector_motion, oav, smargon, synchrotron, test_config_files, dcm): magic_mock = MagicMock(autospec=True) + + with DeviceCollector(mock=True): + magic_mock.zocalo = ZocaloResults() + magic_mock.zebra_fast_grid_scan = ZebraFastGridScan("preifx", "fake_fgs") + magic_mock.oav = oav magic_mock.smargon = smargon magic_mock.detector_motion = detector_motion - magic_mock.zocalo = make_fake_device(ZocaloResults)() magic_mock.dcm = dcm - scan = make_fake_device(ZebraFastGridScan)("prefix", name="fake_fgs") - magic_mock.zebra_fast_grid_scan = scan magic_mock.synchrotron = synchrotron oav.zoom_controller.frst.set("7.5x") oav.parameters = OAVConfigParams( diff --git a/tests/unit_tests/experiment_plans/test_flyscan_xray_centre_plan.py b/tests/unit_tests/experiment_plans/test_flyscan_xray_centre_plan.py index a75a508c4..d62f2b8c0 100644 --- a/tests/unit_tests/experiment_plans/test_flyscan_xray_centre_plan.py +++ b/tests/unit_tests/experiment_plans/test_flyscan_xray_centre_plan.py @@ -152,7 +152,6 @@ def _custom_msg(command_name: str): modified_store_grid_scan_mock, ) class TestFlyscanXrayCentrePlan: - td: TestData = TestData() def test_eiger2_x_16_detector_specified( @@ -311,7 +310,6 @@ def test_read_hardware_for_ispyb_updates_from_ophyd_devices( @patch( "hyperion.experiment_plans.flyscan_xray_centre_plan.move_x_y_z", autospec=True ) - @pytest.mark.asyncio def test_results_adjusted_and_passed_to_move_xyz( self, move_x_y_z: MagicMock, @@ -583,7 +581,6 @@ def test_waits_for_motion_program( fgs, fake_fgs_composite.eiger, fake_fgs_composite.synchrotron, - "zocalo environment", [ test_fgs_params.scan_points_first_grid, test_fgs_params.scan_points_second_grid, @@ -599,7 +596,6 @@ def test_waits_for_motion_program( fgs, fake_fgs_composite.eiger, fake_fgs_composite.synchrotron, - "zocalo environment", [ test_fgs_params.scan_points_first_grid, test_fgs_params.scan_points_second_grid, @@ -1055,11 +1051,13 @@ def test_kickoff_and_complete_gridscan_triggers_zocalo( assert isinstance(zocalo_cb := ispyb_cb.emit_cb, ZocaloCallback) zocalo_env = "dev_env" - zocalo_cb.start({CONST.TRIGGER.ZOCALO: CONST.PLAN.DO_FGS}) # type: ignore - assert zocalo_cb.triggering_plan == CONST.PLAN.DO_FGS - mock_zocalo_trigger_class.return_value = (mock_zocalo_trigger := MagicMock()) + zocalo_cb.start( + {CONST.TRIGGER.ZOCALO: CONST.PLAN.DO_FGS, "zocalo_environment": zocalo_env} # type: ignore + ) + assert zocalo_cb.triggering_plan == CONST.PLAN.DO_FGS + fake_fgs_composite.eiger.unstage = MagicMock() fake_fgs_composite.eiger.odin.file_writer.id.sim_put("test/filename") # type: ignore @@ -1071,7 +1069,6 @@ def test_kickoff_and_complete_gridscan_triggers_zocalo( fake_fgs_composite.zebra_fast_grid_scan, fake_fgs_composite.eiger, fake_fgs_composite.synchrotron, - zocalo_env, scan_points=create_dummy_scan_spec(x_steps, y_steps, z_steps), scan_start_indices=[0, x_steps * y_steps], ) diff --git a/tests/unit_tests/experiment_plans/test_grid_detection_plan.py b/tests/unit_tests/experiment_plans/test_grid_detection_plan.py index 30c5515b4..209d61cb6 100644 --- a/tests/unit_tests/experiment_plans/test_grid_detection_plan.py +++ b/tests/unit_tests/experiment_plans/test_grid_detection_plan.py @@ -113,7 +113,6 @@ def decorated(): lambda a, b: True, ) @patch("bluesky.plan_stubs.sleep", new=MagicMock()) -@pytest.mark.asyncio async def test_grid_detection_plan_gives_warning_error_if_tip_not_found( RE, test_config_files, diff --git a/tests/unit_tests/experiment_plans/test_multi_rotation_scan_plan.py b/tests/unit_tests/experiment_plans/test_multi_rotation_scan_plan.py new file mode 100644 index 000000000..ef6c4076f --- /dev/null +++ b/tests/unit_tests/experiment_plans/test_multi_rotation_scan_plan.py @@ -0,0 +1,440 @@ +from __future__ import annotations + +import json +import shutil +from itertools import takewhile +from math import ceil +from typing import Any, Callable, Sequence +from unittest.mock import MagicMock, patch + +import h5py +import numpy as np +import pytest +from bluesky.run_engine import RunEngine +from bluesky.simulators import RunEngineSimulator, assert_message_and_return_remaining +from dodal.devices.oav.oav_parameters import OAVParameters +from dodal.devices.synchrotron import SynchrotronMode +from ophyd_async.core import set_mock_value + +from hyperion.experiment_plans.rotation_scan_plan import ( + RotationScanComposite, + calculate_motion_profile, + multi_rotation_scan, +) +from hyperion.external_interaction.callbacks.rotation.ispyb_callback import ( + RotationISPyBCallback, +) +from hyperion.external_interaction.callbacks.rotation.nexus_callback import ( + RotationNexusFileCallback, +) +from hyperion.external_interaction.ispyb.ispyb_store import StoreInIspyb +from hyperion.parameters.constants import CONST +from hyperion.parameters.rotation import MultiRotationScan, RotationScan + +from ...conftest import ( + DocumentCapturer, + RunEngineSimulator, + extract_metafile, + fake_read, + raw_params_from_file, +) +from ..external_interaction.conftest import * # noqa # for fixtures +from ..external_interaction.conftest import mx_acquisition_from_conn + +TEST_OFFSET = 1 +TEST_SHUTTER_OPENING_DEGREES = 2.5 + + +def test_multi_rotation_scan_params(): + raw_params = raw_params_from_file( + "tests/test_data/parameter_json_files/good_test_multi_rotation_scan_parameters.json" + ) + params = MultiRotationScan(**raw_params) + omega_starts = [s["omega_start_deg"] for s in raw_params["rotation_scans"]] + for i, scan in enumerate(params.single_rotation_scans): + assert scan.omega_start_deg == omega_starts[i] + assert scan.nexus_vds_start_img == params.scan_indices[i] + assert params.scan_indices + + +async def test_multi_rotation_plan_runs_multiple_plans_in_one_arm( + fake_create_rotation_devices: RotationScanComposite, + test_multi_rotation_params: MultiRotationScan, + sim_run_engine_for_rotation: RunEngineSimulator, + oav_parameters_for_rotation: OAVParameters, +): + smargon = fake_create_rotation_devices.smargon + omega = smargon.omega + set_mock_value( + fake_create_rotation_devices.synchrotron.synchrotron_mode, SynchrotronMode.USER + ) + msgs = sim_run_engine_for_rotation.simulate_plan( + multi_rotation_scan( + fake_create_rotation_devices, + test_multi_rotation_params, + oav_parameters_for_rotation, + ) + ) + + msgs = assert_message_and_return_remaining( + msgs, lambda msg: msg.command == "stage" and msg.obj.name == "eiger" + )[1:] + + msgs_within_arming = list( + takewhile( + lambda msg: msg.command != "unstage" + and (not msg.obj or msg.obj.name != "eiger"), + msgs, + ) + ) + + def _assert_set_seq_and_return_remaining(remaining, name_value_pairs): + for name, value in name_value_pairs: + try: + remaining = assert_message_and_return_remaining( + remaining, + lambda msg: msg.command == "set" + and msg.obj.name == name + and msg.args == (value,), + ) + except Exception as e: + raise Exception(f"Failed to find {name} being set to {value}") from e + return remaining + + for scan in test_multi_rotation_params.single_rotation_scans: + motion_values = calculate_motion_profile( + scan, + (await omega.acceleration_time.get_value()), + (await omega.max_velocity.get_value()), + ) + # moving to the start position + msgs_within_arming = _assert_set_seq_and_return_remaining( + msgs_within_arming, + [ + ("smargon-x", scan.x_start_um / 1000), # type: ignore + ("smargon-y", scan.y_start_um / 1000), # type: ignore + ("smargon-z", scan.z_start_um / 1000), # type: ignore + ("smargon-phi", scan.phi_start_deg), + ("smargon-chi", scan.chi_start_deg), + ], + ) + # arming the zebra + msgs_within_arming = assert_message_and_return_remaining( + msgs_within_arming, + lambda msg: msg.command == "set" and msg.obj.name == "zebra-pc-arm", + ) + # the final rel_set of omega to trigger the scan + assert_message_and_return_remaining( + msgs_within_arming, + lambda msg: msg.command == "set" + and msg.obj.name == "smargon-omega" + and msg.args + == ( + (scan.scan_width_deg + motion_values.shutter_opening_deg) + * motion_values.direction.multiplier, + ), + ) + + +def _run_multi_rotation_plan( + RE: RunEngine, + params: MultiRotationScan, + devices: RotationScanComposite, + callbacks: Sequence[Callable[[str, dict[str, Any]], Any]], + oav_params: OAVParameters, +): + for cb in callbacks: + RE.subscribe(cb) + with patch("bluesky.preprocessors.__read_and_stash_a_motor", fake_read): + RE(multi_rotation_scan(devices, params, oav_params)) + + +def test_full_multi_rotation_plan_docs_emitted( + RE: RunEngine, + test_multi_rotation_params: MultiRotationScan, + fake_create_rotation_devices: RotationScanComposite, + oav_parameters_for_rotation: OAVParameters, +): + callback_sim = DocumentCapturer() + _run_multi_rotation_plan( + RE, + test_multi_rotation_params, + fake_create_rotation_devices, + [callback_sim], + oav_parameters_for_rotation, + ) + docs = callback_sim.docs_received + + assert ( + outer_plan_start_doc := DocumentCapturer.assert_doc( + docs, "start", matches_fields=({"plan_name": "multi_rotation_scan"}) + ) + ) + outer_uid = outer_plan_start_doc[1]["uid"] + inner_run_docs = DocumentCapturer.get_docs_until( + docs, + "stop", + matches_fields=({"run_start": outer_uid, "exit_status": "success"}), + )[1:-1] + + for scan in test_multi_rotation_params.single_rotation_scans: + inner_run_docs = DocumentCapturer.get_docs_from( + inner_run_docs, + "start", + matches_fields={"subplan_name": "rotation_scan_with_cleanup"}, + ) + scan_docs = DocumentCapturer.get_docs_until( + inner_run_docs, + "stop", + matches_fields={"run_start": inner_run_docs[0][1]["uid"]}, + ) + assert DocumentCapturer.is_match( + scan_docs[0], + "start", + has_fields=["trigger_zocalo_on", "hyperion_parameters"], + ) + params = RotationScan(**json.loads(scan_docs[0][1]["hyperion_parameters"])) + assert params == scan + assert len(events := DocumentCapturer.get_matches(scan_docs, "event")) == 3 + DocumentCapturer.assert_events_and_data_in_order( + events, + [ + ["eiger_odin_file_writer_id"], + ["undulator-current_gap", "synchrotron-synchrotron_mode", "smargon-x"], + [ + "attenuator-actual_transmission", + "flux_flux_reading", + "dcm-energy_in_kev", + "eiger_bit_depth", + ], + ], + ) + inner_run_docs = DocumentCapturer.get_docs_from( + inner_run_docs, + "stop", + matches_fields={"run_start": inner_run_docs[0][1]["uid"]}, + ) + + +@patch("hyperion.external_interaction.callbacks.rotation.nexus_callback.NexusWriter") +def test_full_multi_rotation_plan_nexus_writer_called_correctly( + mock_nexus_writer: MagicMock, + RE: RunEngine, + test_multi_rotation_params: MultiRotationScan, + fake_create_rotation_devices: RotationScanComposite, + oav_parameters_for_rotation: OAVParameters, +): + callback = RotationNexusFileCallback() + _run_multi_rotation_plan( + RE, + test_multi_rotation_params, + fake_create_rotation_devices, + [callback], + oav_parameters_for_rotation, + ) + nexus_writer_calls = mock_nexus_writer.call_args_list + first_run_number = test_multi_rotation_params.detector_params.run_number + for call, rotation_params in zip( + nexus_writer_calls, test_multi_rotation_params.single_rotation_scans + ): + assert call.args[0] == rotation_params + assert call.kwargs == { + "omega_start_deg": rotation_params.omega_start_deg, + "chi_start_deg": rotation_params.chi_start_deg, + "phi_start_deg": rotation_params.phi_start_deg, + "vds_start_index": rotation_params.nexus_vds_start_img, + "full_num_of_images": test_multi_rotation_params.num_images, + "meta_data_run_number": first_run_number, + "rotation_direction": rotation_params.rotation_direction, + } + + +def test_full_multi_rotation_plan_nexus_files_written_correctly( + RE: RunEngine, + test_multi_rotation_params: MultiRotationScan, + fake_create_rotation_devices: RotationScanComposite, + oav_parameters_for_rotation: OAVParameters, + tmpdir, +): + multi_params = test_multi_rotation_params + prefix = "multi_rotation_test" + test_data_dir = "tests/test_data/nexus_files/" + meta_file = f"{test_data_dir}rotation/ins_8_5_meta.h5.gz" + fake_datafile = f"{test_data_dir}fake_data.h5" + multi_params.file_name = prefix + multi_params.storage_directory = f"{tmpdir}" + meta_data_run_number = multi_params.detector_params.run_number + + data_filename_prefix = f"{prefix}_{meta_data_run_number}_" + meta_filename = f"{prefix}_{meta_data_run_number}_meta.h5" + + callback = RotationNexusFileCallback() + _run_multi_rotation_plan( + RE, + multi_params, + fake_create_rotation_devices, + [callback], + oav_parameters_for_rotation, + ) + + def _expected_dset_number(image_number: int): + # image numbers 0-999 are in dset 1, etc. + return int(ceil((image_number + 1) / 1000)) + + num_datasets = range( + 1, _expected_dset_number(multi_params.num_images - 1) + ) # the index of the last image is num_images - 1 + + for i in num_datasets: + shutil.copy( + fake_datafile, + f"{tmpdir}/{data_filename_prefix}{i:06d}.h5", + ) + extract_metafile( + meta_file, + f"{tmpdir}/{meta_filename}", + ) + for i, scan in enumerate(multi_params.single_rotation_scans): + with h5py.File(f"{tmpdir}/{prefix}_{i+1}.nxs", "r") as written_nexus_file: + # check links go to the right file: + detector_specific = written_nexus_file[ + "entry/instrument/detector/detectorSpecific" + ] + for field in ["software_version"]: + link = detector_specific.get(field, getlink=True) # type: ignore + assert link.filename == meta_filename # type: ignore + data_group = written_nexus_file["entry/data"] + for field in [f"data_{n:06d}" for n in num_datasets]: + link = data_group.get(field, getlink=True) # type: ignore + assert link.filename.startswith(data_filename_prefix) # type: ignore + + # check dataset starts and stops are correct: + assert isinstance(dataset := data_group["data"], h5py.Dataset) # type: ignore + assert dataset.is_virtual + assert dataset[scan.num_images - 1, 0, 0] == 0 + with pytest.raises(IndexError): + assert dataset[scan.num_images, 0, 0] == 0 + dataset_sources = dataset.virtual_sources() + expected_dset_start = _expected_dset_number(multi_params.scan_indices[i]) + expected_dset_end = _expected_dset_number(multi_params.scan_indices[i + 1]) + dset_start_name = dataset_sources[0].dset_name + dset_end_name = dataset_sources[-1].dset_name + assert dset_start_name.endswith(f"data_{expected_dset_start:06d}") + assert dset_end_name.endswith(f"data_{expected_dset_end:06d}") + + # check scan values are correct for each file: + assert isinstance( + chi := written_nexus_file["/entry/sample/sample_chi/chi"], h5py.Dataset + ) + assert chi[:] == scan.chi_start_deg + assert isinstance( + phi := written_nexus_file["/entry/sample/sample_phi/phi"], h5py.Dataset + ) + assert phi[:] == scan.phi_start_deg + assert isinstance( + omega := written_nexus_file["/entry/sample/sample_omega/omega"], + h5py.Dataset, + ) + omega = omega[:] + assert isinstance( + omega_end := written_nexus_file["/entry/sample/sample_omega/omega_end"], + h5py.Dataset, + ) + omega_end = omega_end[:] + assert len(omega) == scan.num_images + expected_omega_starts = np.linspace( + scan.omega_start_deg, + scan.omega_start_deg + + ((scan.num_images - 1) * multi_params.rotation_increment_deg), + scan.num_images, + ) + assert np.allclose(omega, expected_omega_starts) + expected_omega_ends = ( + expected_omega_starts + multi_params.rotation_increment_deg + ) + assert np.allclose(omega_end, expected_omega_ends) + assert isinstance( + omega_transform := written_nexus_file[ + "/entry/sample/transformations/omega" + ], + h5py.Dataset, + ) + assert isinstance(omega_vec := omega_transform.attrs["vector"], np.ndarray) + assert tuple(omega_vec) == (1.0 * scan.rotation_direction.multiplier, 0, 0) + + +@patch("hyperion.external_interaction.callbacks.rotation.ispyb_callback.StoreInIspyb") +def test_full_multi_rotation_plan_ispyb_called_correctly( + mock_ispyb_store: MagicMock, + RE: RunEngine, + test_multi_rotation_params: MultiRotationScan, + fake_create_rotation_devices: RotationScanComposite, + oav_parameters_for_rotation: OAVParameters, +): + callback = RotationISPyBCallback() + mock_ispyb_store.return_value = MagicMock(spec=StoreInIspyb) + _run_multi_rotation_plan( + RE, + test_multi_rotation_params, + fake_create_rotation_devices, + [callback], + oav_parameters_for_rotation, + ) + ispyb_calls = mock_ispyb_store.call_args_list + for instantiation_call, ispyb_store_calls, rotation_params in zip( + ispyb_calls, + [ # there should be 4 calls to the IspybStore per run + mock_ispyb_store.return_value.method_calls[i * 4 : (i + 1) * 4] + for i in range(len(test_multi_rotation_params.rotation_scans)) + ], + test_multi_rotation_params.single_rotation_scans, + ): + assert instantiation_call.args[0] == CONST.SIM.ISPYB_CONFIG + assert ispyb_store_calls[0][0] == "begin_deposition" + assert ispyb_store_calls[1][0] == "update_deposition" + assert ispyb_store_calls[2][0] == "update_deposition" + assert ispyb_store_calls[3][0] == "end_deposition" + + +def test_full_multi_rotation_plan_ispyb_interaction_end_to_end( + mock_ispyb_conn_multiscan, + RE: RunEngine, + test_multi_rotation_params: MultiRotationScan, + fake_create_rotation_devices: RotationScanComposite, + oav_parameters_for_rotation: OAVParameters, +): + number_of_scans = len(test_multi_rotation_params.rotation_scans) + callback = RotationISPyBCallback() + _run_multi_rotation_plan( + RE, + test_multi_rotation_params, + fake_create_rotation_devices, + [callback], + oav_parameters_for_rotation, + ) + mx = mx_acquisition_from_conn(mock_ispyb_conn_multiscan) + assert mx.get_data_collection_group_params.call_count == number_of_scans + assert mx.get_data_collection_params.call_count == number_of_scans * 4 + for upsert_calls, rotation_params in zip( + [ # there should be 4 datacollection upserts per scan + mx.upsert_data_collection.call_args_list[i * 4 : (i + 1) * 4] + for i in range(len(test_multi_rotation_params.rotation_scans)) + ], + test_multi_rotation_params.single_rotation_scans, + ): + first_upsert_data = upsert_calls[0].args[0] + assert ( + first_upsert_data[12] - first_upsert_data[11] + == rotation_params.scan_width_deg + ) + assert first_upsert_data[15] == rotation_params.num_images + second_upsert_data = upsert_calls[1].args[0] + assert second_upsert_data[29].startswith("Sample position") + position_string = f"{rotation_params.x_start_um:.0f}, {rotation_params.y_start_um:.0f}, {rotation_params.z_start_um:.0f}" + assert position_string in second_upsert_data[29] + third_upsert_data = upsert_calls[2].args[0] + assert third_upsert_data[24] > 0 # resolution + assert third_upsert_data[52] > 0 # beam size + fourth_upsert_data = upsert_calls[3].args[0] + assert fourth_upsert_data[9] # timestamp + assert fourth_upsert_data[10] == "DataCollection Successful" diff --git a/tests/unit_tests/experiment_plans/test_pin_centre_then_xray_centre_plan.py b/tests/unit_tests/experiment_plans/test_pin_centre_then_xray_centre_plan.py index 744b55d3f..9577cdeb0 100644 --- a/tests/unit_tests/experiment_plans/test_pin_centre_then_xray_centre_plan.py +++ b/tests/unit_tests/experiment_plans/test_pin_centre_then_xray_centre_plan.py @@ -135,7 +135,7 @@ def add_handlers_to_simulate_detector_motion(msg: Msg): assert messages[0].args[0] == 100 assert messages[0].kwargs["group"] == CONST.WAIT.GRID_READY_FOR_DC assert messages[1].obj is simple_beamline.detector_motion.shutter - assert messages[1].args[0] == 1 + assert messages[1].args[0] == ShutterState.OPEN assert messages[1].kwargs["group"] == CONST.WAIT.GRID_READY_FOR_DC messages = assert_message_and_return_remaining( messages[2:], diff --git a/tests/unit_tests/experiment_plans/test_rotation_scan_plan.py b/tests/unit_tests/experiment_plans/test_rotation_scan_plan.py index 6068e2ac6..ab0b2a901 100644 --- a/tests/unit_tests/experiment_plans/test_rotation_scan_plan.py +++ b/tests/unit_tests/experiment_plans/test_rotation_scan_plan.py @@ -203,9 +203,9 @@ async def test_full_rotation_plan_smargon_settings( assert await smargon.phi.user_readback.get_value() == params.phi_start_deg assert await smargon.chi.user_readback.get_value() == params.chi_start_deg - assert await smargon.x.user_readback.get_value() == params.x_start_um - assert await smargon.y.user_readback.get_value() == params.y_start_um - assert await smargon.z.user_readback.get_value() == params.z_start_um + assert await smargon.x.user_readback.get_value() == params.x_start_um / 1000 # type: ignore + assert await smargon.y.user_readback.get_value() == params.y_start_um / 1000 # type: ignore + assert await smargon.z.user_readback.get_value() == params.z_start_um / 1000 # type: ignore assert ( # 4 * snapshots, restore omega, 1 * rotation sweep omega_set.call_count == 4 + 1 + 1 @@ -229,7 +229,7 @@ async def test_rotation_plan_moves_aperture_correctly( ) assert aperture_scatterguard.aperture_positions assert ( - await aperture_scatterguard._get_current_aperture_position() + await aperture_scatterguard.get_current_aperture_position() == aperture_scatterguard.aperture_positions.SMALL ) @@ -251,7 +251,7 @@ async def test_rotation_plan_smargon_doesnt_move_xyz_if_not_given_in_params( get_mock_put(motor.user_setpoint).assert_not_called() # type: ignore -@patch("hyperion.experiment_plans.rotation_scan_plan.cleanup_plan", autospec=True) +@patch("hyperion.experiment_plans.rotation_scan_plan._cleanup_plan", autospec=True) @patch("bluesky.plan_stubs.wait", autospec=True) def test_cleanup_happens( bps_wait: MagicMock, @@ -294,12 +294,14 @@ class MyTestException(Exception): def test_rotation_plan_reads_hardware( RE: RunEngine, fake_create_rotation_devices: RotationScanComposite, - test_rotation_params: RotationScan, - motion_values: RotationMotionProfile, - sim_run_engine: RunEngineSimulator, + test_rotation_params, + motion_values, + sim_run_engine_for_rotation: RunEngineSimulator, ): - _add_sim_handlers_for_normal_operation(fake_create_rotation_devices, sim_run_engine) - msgs = sim_run_engine.simulate_plan( + _add_sim_handlers_for_normal_operation( + fake_create_rotation_devices, sim_run_engine_for_rotation + ) + msgs = sim_run_engine_for_rotation.simulate_plan( rotation_scan_plan( fake_create_rotation_devices, test_rotation_params, motion_values ) @@ -341,14 +343,14 @@ def test_rotation_scan_initialises_detector_distance_shutter_and_tx_fraction( msgs, lambda msg: msg.command == "set" and msg.args[0] == 1 - and msg.obj.name == "detector_motion_shutter" + and msg.obj.name == "detector_motion-shutter" and msg.kwargs["group"] == "setup_senv", ) msgs = assert_message_and_return_remaining( msgs, lambda msg: msg.command == "set" and msg.args[0] == test_rotation_params.detector_distance_mm - and msg.obj.name == "detector_motion_z" + and msg.obj.name == "detector_motion-z" and msg.kwargs["group"] == "setup_senv", ) msgs = assert_message_and_return_remaining( @@ -387,7 +389,7 @@ def test_rotation_scan_moves_gonio_to_start_before_snapshots( msgs = assert_message_and_return_remaining( msgs, lambda msg: msg.command == "wait" - and msg.kwargs["group"] == "move_gonio_to_start", + and msg.kwargs["group"] == CONST.WAIT.MOVE_GONIO_TO_START, ) msgs = assert_message_and_return_remaining( msgs, diff --git a/tests/unit_tests/experiment_plans/test_wait_for_robot_load_then_centre.py b/tests/unit_tests/experiment_plans/test_wait_for_robot_load_then_centre.py index 43d1d4ae3..66915a111 100644 --- a/tests/unit_tests/experiment_plans/test_wait_for_robot_load_then_centre.py +++ b/tests/unit_tests/experiment_plans/test_wait_for_robot_load_then_centre.py @@ -7,11 +7,10 @@ from bluesky.simulators import RunEngineSimulator, assert_message_and_return_remaining from bluesky.utils import Msg from dodal.devices.aperturescatterguard import AperturePositions -from dodal.devices.eiger import EigerDetector from dodal.devices.oav.oav_detector import OAV -from dodal.devices.smargon import Smargon, StubPosition +from dodal.devices.smargon import StubPosition from dodal.devices.webcam import Webcam -from ophyd.sim import NullStatus, instantiate_fake_device +from ophyd.sim import NullStatus from ophyd_async.core import set_mock_value from hyperion.experiment_plans.robot_load_then_centre_plan import ( @@ -30,7 +29,7 @@ @pytest.fixture def robot_load_composite( - smargon, dcm, robot, aperture_scatterguard, oav, webcam, thawer, lower_gonio + smargon, dcm, robot, aperture_scatterguard, oav, webcam, thawer, lower_gonio, eiger ) -> RobotLoadThenCentreComposite: composite: RobotLoadThenCentreComposite = MagicMock() composite.smargon = smargon @@ -44,6 +43,7 @@ def robot_load_composite( composite.webcam = webcam composite.lower_gonio = lower_gonio composite.thawer = thawer + composite.eiger = eiger return composite @@ -131,6 +131,7 @@ def test_robot_load_then_centre_doesnt_set_energy_if_not_specified_and_current_e robot_load_then_centre_params_no_energy: RobotLoadThenCentre, sim_run_engine: RunEngineSimulator, ): + robot_load_composite.eiger.set_detector_parameters = MagicMock() sim_run_engine.add_handler( "read", lambda msg: {"dcm-energy_in_kev": {"value": 11.105}}, @@ -153,9 +154,6 @@ def run_simulating_smargon_wait( total_disabled_reads, sim_run_engine: RunEngineSimulator, ): - robot_load_composite.smargon = instantiate_fake_device(Smargon, name="smargon") - robot_load_composite.eiger = instantiate_fake_device(EigerDetector, name="eiger") - num_of_reads = 0 def return_not_disabled_after_reads(_): diff --git a/tests/unit_tests/external_interaction/callbacks/conftest.py b/tests/unit_tests/external_interaction/callbacks/conftest.py index ab934f77e..4cb81f67a 100644 --- a/tests/unit_tests/external_interaction/callbacks/conftest.py +++ b/tests/unit_tests/external_interaction/callbacks/conftest.py @@ -71,6 +71,7 @@ class TestData(OavGridSnapshotTestEvents): test_rotation_start_main_document = { "uid": "2093c941-ded1-42c4-ab74-ea99980fbbfd", "subplan_name": CONST.PLAN.ROTATION_MAIN, + "zocalo_environment": "dev_artemis", } test_gridscan_outer_start_document = { "uid": "d8bee3ee-f614-4e7a-a516-25d6b9e87ef3", @@ -80,6 +81,7 @@ class TestData(OavGridSnapshotTestEvents): "plan_type": "generator", "plan_name": CONST.PLAN.GRIDSCAN_OUTER, "subplan_name": CONST.PLAN.GRIDSCAN_OUTER, + "zocalo_environment": "dev_artemis", CONST.TRIGGER.ZOCALO: CONST.PLAN.DO_FGS, "hyperion_parameters": dummy_params().json(), } @@ -127,7 +129,6 @@ class TestData(OavGridSnapshotTestEvents): "plan_type": "generator", "plan_name": CONST.PLAN.GRIDSCAN_AND_MOVE, "subplan_name": CONST.PLAN.DO_FGS, - "zocalo_environment": "dev_artemis", "scan_points": create_dummy_scan_spec(10, 20, 30), } test_descriptor_document_oav_rotation_snapshot: EventDescriptor = { diff --git a/tests/unit_tests/external_interaction/callbacks/test_rotation_callbacks.py b/tests/unit_tests/external_interaction/callbacks/test_rotation_callbacks.py index b47c6db29..f8efd8967 100644 --- a/tests/unit_tests/external_interaction/callbacks/test_rotation_callbacks.py +++ b/tests/unit_tests/external_interaction/callbacks/test_rotation_callbacks.py @@ -103,6 +103,7 @@ def fake_rotation_scan( "subplan_name": CONST.PLAN.ROTATION_OUTER, "hyperion_parameters": params.json(), CONST.TRIGGER.ZOCALO: CONST.PLAN.ROTATION_MAIN, + "zocalo_environment": params.zocalo_environment, } ) def plan(): @@ -173,7 +174,7 @@ def test_nexus_handler_only_writes_once( params: RotationScan, test_outer_start_doc, ): - nexus_writer.return_value.full_filename = "test_full_filename" + nexus_writer.return_value.data_filename = "test_full_filename" cb = RotationNexusFileCallback() cb.active = True RE(fake_rotation_scan(params, [cb])) @@ -222,7 +223,7 @@ def test_zocalo_start_and_end_not_triggered_if_ispyb_ids_not_present( params: RotationScan, test_outer_start_doc, ): - nexus_writer.return_value.full_filename = "test_full_filename" + nexus_writer.return_value.data_filename = "test_full_filename" nexus_callback, ispyb_callback = create_rotation_callbacks() activate_callbacks((nexus_callback, ispyb_callback)) @@ -257,7 +258,7 @@ def test_ispyb_starts_on_opening_and_zocalo_on_main_so_ispyb_triggered_before_zo mock_store_in_ispyb_instance.update_deposition.return_value = returned_ids ispyb_store.return_value = mock_store_in_ispyb_instance - nexus_writer.return_value.full_filename = "test_full_filename" + nexus_writer.return_value.data_filename = "test_full_filename" nexus_callback, ispyb_callback = create_rotation_callbacks() activate_callbacks((nexus_callback, ispyb_callback)) ispyb_callback.emit_cb.stop = MagicMock() # type: ignore diff --git a/tests/unit_tests/external_interaction/callbacks/test_zocalo_handler.py b/tests/unit_tests/external_interaction/callbacks/test_zocalo_handler.py index 6d2bfbd39..73be97d5f 100644 --- a/tests/unit_tests/external_interaction/callbacks/test_zocalo_handler.py +++ b/tests/unit_tests/external_interaction/callbacks/test_zocalo_handler.py @@ -24,13 +24,17 @@ td = TestData() +def start_dict(plan_name: str = "test_plan_name", env: str = "test_env"): + return {CONST.TRIGGER.ZOCALO: plan_name, "zocalo_environment": env} + + class TestZocaloHandler: def _setup_handler(self): zocalo_handler = ZocaloCallback() assert zocalo_handler.triggering_plan is None - zocalo_handler.start({CONST.TRIGGER.ZOCALO: "test_plan_name"}) # type: ignore + zocalo_handler.start(start_dict()) # type: ignore assert zocalo_handler.triggering_plan == "test_plan_name" - assert zocalo_handler.zocalo_interactor is None + assert zocalo_handler.zocalo_interactor is not None return zocalo_handler def test_handler_gets_plan_name_from_start_doc(self): @@ -38,17 +42,15 @@ def test_handler_gets_plan_name_from_start_doc(self): def test_handler_doesnt_trigger_on_wrong_plan(self): zocalo_handler = self._setup_handler() - zocalo_handler.start({CONST.TRIGGER.ZOCALO: "_not_test_plan_name"}) # type: ignore + zocalo_handler.start(start_dict("_not_test_plan_name")) # type: ignore def test_handler_raises_on_right_plan_with_wrong_metadata(self): zocalo_handler = self._setup_handler() - assert zocalo_handler.zocalo_interactor is None with pytest.raises(AssertionError): zocalo_handler.start({"subplan_name": "test_plan_name"}) # type: ignore def test_handler_raises_on_right_plan_with_no_ispyb_ids(self): zocalo_handler = self._setup_handler() - assert zocalo_handler.zocalo_interactor is None with pytest.raises(ISPyBDepositionNotMade): zocalo_handler.start( { @@ -64,7 +66,6 @@ def test_handler_raises_on_right_plan_with_no_ispyb_ids(self): ) def test_handler_inits_zocalo_trigger_on_right_plan(self, zocalo_trigger): zocalo_handler = self._setup_handler() - assert zocalo_handler.zocalo_interactor is None zocalo_handler.start( { "subplan_name": "test_plan_name", diff --git a/tests/unit_tests/external_interaction/conftest.py b/tests/unit_tests/external_interaction/conftest.py index f4fa60ae7..978d6c292 100644 --- a/tests/unit_tests/external_interaction/conftest.py +++ b/tests/unit_tests/external_interaction/conftest.py @@ -95,8 +95,7 @@ def test_fgs_params(request): yield params -@pytest.fixture -def mock_ispyb_conn(base_ispyb_conn): +def _mock_ispyb_conn(base_ispyb_conn, position_id, dcgid, dcids, giids): def upsert_data_collection(values): kvpairs = remap_upsert_columns( list(MXAcquisition.get_data_collection_params()), values @@ -106,12 +105,10 @@ def upsert_data_collection(values): else: return next(upsert_data_collection.i) # pyright: ignore - upsert_data_collection.i = iter(TEST_DATA_COLLECTION_IDS) # pyright: ignore - mx_acq = base_ispyb_conn.return_value.mx_acquisition mx_acq.upsert_data_collection.side_effect = upsert_data_collection - mx_acq.update_dc_position.return_value = TEST_POSITION_ID - mx_acq.upsert_data_collection_group.return_value = TEST_DATA_COLLECTION_GROUP_ID + mx_acq.update_dc_position.return_value = position_id + mx_acq.upsert_data_collection_group.return_value = dcgid def upsert_dc_grid(values): kvpairs = remap_upsert_columns(list(MXAcquisition.get_dc_grid_params()), values) @@ -120,12 +117,35 @@ def upsert_dc_grid(values): else: return next(upsert_dc_grid.i) # pyright: ignore - upsert_dc_grid.i = iter(TEST_GRID_INFO_IDS) # pyright: ignore + upsert_data_collection.i = iter(dcids) # pyright: ignore + upsert_dc_grid.i = iter(giids) # pyright: ignore mx_acq.upsert_dc_grid.side_effect = upsert_dc_grid return base_ispyb_conn +@pytest.fixture +def mock_ispyb_conn(base_ispyb_conn): + return _mock_ispyb_conn( + base_ispyb_conn, + TEST_POSITION_ID, + TEST_DATA_COLLECTION_GROUP_ID, + TEST_DATA_COLLECTION_IDS, + TEST_GRID_INFO_IDS, + ) + + +@pytest.fixture +def mock_ispyb_conn_multiscan(base_ispyb_conn): + return _mock_ispyb_conn( + base_ispyb_conn, + TEST_POSITION_ID, + TEST_DATA_COLLECTION_GROUP_ID, + list(range(12, 24)), + list(range(56, 68)), + ) + + def mx_acquisition_from_conn(mock_ispyb_conn) -> MagicMock: return mock_ispyb_conn.return_value.__enter__.return_value.mx_acquisition diff --git a/tests/unit_tests/external_interaction/test_write_rotation_nexus.py b/tests/unit_tests/external_interaction/test_write_rotation_nexus.py index c57181d36..01b544e12 100644 --- a/tests/unit_tests/external_interaction/test_write_rotation_nexus.py +++ b/tests/unit_tests/external_interaction/test_write_rotation_nexus.py @@ -17,6 +17,7 @@ from hyperion.external_interaction.callbacks.rotation.nexus_callback import ( RotationNexusFileCallback, ) +from hyperion.external_interaction.nexus.write_nexus import NexusWriter from hyperion.log import LOGGER from hyperion.parameters.constants import CONST from hyperion.parameters.rotation import RotationScan @@ -470,3 +471,12 @@ def _compare_actual_and_expected(path: list[str], actual, expected, exceptions: actual_value, expected_value, # type: ignore ), f"Actual and expected values differ for {item_path_str}: {actual_value_str} != {expected_value_str}" + + +def test_override_parameters_override(test_params: RotationScan): + writer = NexusWriter( + test_params, (1, 2, 3), {}, full_num_of_images=82367, meta_data_run_number=9852 + ) + assert writer.full_num_of_images != test_params.num_images + assert writer.full_num_of_images == 82367 + assert writer.data_filename == f"{test_params.file_name}_9852" diff --git a/tests/unit_tests/test_exceptions.py b/tests/unit_tests/test_exceptions.py index 304736fa3..165fea147 100644 --- a/tests/unit_tests/test_exceptions.py +++ b/tests/unit_tests/test_exceptions.py @@ -4,20 +4,20 @@ from hyperion.exceptions import WarningException, catch_exception_and_warn -class TestException(Exception): +class _TestException(Exception): pass def dummy_plan(): yield from null() - raise TestException + raise _TestException def test_catch_exception_and_warn_correctly_raises_warning_exception(RE): with pytest.raises(WarningException): - RE(catch_exception_and_warn(TestException, dummy_plan)) + RE(catch_exception_and_warn(_TestException, dummy_plan)) def test_catch_exception_and_warn_correctly_raises_original_exception(RE): - with pytest.raises(TestException): + with pytest.raises(_TestException): RE(catch_exception_and_warn(ValueError, dummy_plan)) diff --git a/tests/unit_tests/utils/test_callback_sim.py b/tests/unit_tests/utils/test_callback_sim.py new file mode 100644 index 000000000..30db7d9e0 --- /dev/null +++ b/tests/unit_tests/utils/test_callback_sim.py @@ -0,0 +1,51 @@ +import pytest + +from ...conftest import DocumentCapturer + + +@pytest.fixture +def test_docs(): + return [ + ("start", {"uid": 12345, "abc": 56789, "xyz": 99999}), + ("stop", {"uid": 77777, "abc": 88888, "xyz": 99999}), + ] + + +def test_callback_sim_doc_names(test_docs): + DocumentCapturer.assert_doc(test_docs, "start") + DocumentCapturer.assert_doc(test_docs, "stop") + DocumentCapturer.assert_doc(test_docs, "restart", does_exist=False) + + +def test_callback_sim_has_fields(test_docs): + DocumentCapturer.assert_doc(test_docs, "start", has_fields=["uid"]) + DocumentCapturer.assert_doc(test_docs, "stop", has_fields=["abc", "xyz"]) + DocumentCapturer.assert_doc( + test_docs, "start", has_fields=["uid", "bbb"], does_exist=False + ) + + +def test_callback_sim_matches_fields(test_docs): + DocumentCapturer.assert_doc(test_docs, "start", matches_fields={"uid": 12345}) + DocumentCapturer.assert_doc( + test_docs, "stop", matches_fields={"abc": 88888, "xyz": 99999} + ) + DocumentCapturer.assert_doc( + test_docs, + "start", + matches_fields={"abc": 88888, "xyz": 99799}, + does_exist=False, + ) + + +def test_callback_sim_assert_switch(test_docs): + with pytest.raises(AssertionError): + DocumentCapturer.assert_doc(test_docs, "restart") + + with pytest.raises(AssertionError): + DocumentCapturer.assert_doc(test_docs, "start", has_fields=["uid", "bbb"]) + + with pytest.raises(AssertionError): + DocumentCapturer.assert_doc( + test_docs, "start", matches_fields={"abc": 88888, "xyz": 99799} + )