From f7d4c7321b10afa3798149a306e7c4f102a9df07 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Wed, 20 Mar 2024 13:02:32 -0400 Subject: [PATCH 1/6] Working prototype of TiledWriter-based data access --- startup/00-startup.py | 13 ++++++++++++- startup/10-panda.py | 5 ++--- startup/30-handlers.py | 3 ++- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/startup/00-startup.py b/startup/00-startup.py index 5914f4b..ab5f928 100644 --- a/startup/00-startup.py +++ b/startup/00-startup.py @@ -2,28 +2,39 @@ print(f"Loading file {__file__!r} ...") import datetime import logging +import os import subprocess import warnings import nslsii import ophyd.signal from bluesky.callbacks.broker import post_run, verify_files_saved +from bluesky.callbacks.tiled_writer import TiledWriter from bluesky.run_engine import call_in_bluesky_event_loop +from databroker.v0 import Broker from IPython import get_ipython +from tiled.client import from_uri ophyd.signal.EpicsSignal.set_defaults(connection_timeout=5) # See docstring for nslsii.configure_base() for more details # this command takes away much of the boilerplate for settting up a profile # (such as setting up best effort callbacks etc) + + nslsii.configure_base( get_ipython().user_ns, - "tst", + Broker.named("temp"), pbar=True, bec=True, magics=True, mpl=True, epics_context=False, ) +RE.unsubscribe(0) # remove temp databroker subscription + +tiled_client = from_uri("http://localhost:8000", api_key=os.getenv("TILED_API_KEY", "")) +tw = TiledWriter(tiled_client) +RE.subscribe(tw) # This is needed for ophyd-async to enable 'await <>' instead of 'asyncio.run(<>)': get_ipython().run_line_magic("autoawait", "call_in_bluesky_event_loop") diff --git a/startup/10-panda.py b/startup/10-panda.py index a61b1dd..b209a63 100644 --- a/startup/10-panda.py +++ b/startup/10-panda.py @@ -1,3 +1,5 @@ +print(f"Loading file {__file__!r} ...") + import asyncio import datetime import json @@ -130,9 +132,6 @@ class BITS(Device): D = Cpt(EpicsSignal, "D") -print(f"Loading file {__file__!r} ...") - - class PandA_Ophyd1(Device): pcap = Cpt(PCAP, "PCAP:") data = Cpt(DATA, "DATA:") diff --git a/startup/30-handlers.py b/startup/30-handlers.py index 1681fe0..f8d8947 100644 --- a/startup/30-handlers.py +++ b/startup/30-handlers.py @@ -19,4 +19,5 @@ def __call__(self, field): return entry[:] -db.reg.register_handler("PANDA", PandAHandlerHDF5, overwrite=True) +# TODO: remove completely when Tiled is used: +# db.reg.register_handler("PANDA", PandAHandlerHDF5, overwrite=True) From d5cb4b6cfa699acfa4c4b96922e9f84004a208ae Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Thu, 21 Mar 2024 17:49:02 -0400 Subject: [PATCH 2/6] Fix for dtypes for panda data access with Tiled --- startup/10-panda.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/startup/10-panda.py b/startup/10-panda.py index b209a63..06386a3 100644 --- a/startup/10-panda.py +++ b/startup/10-panda.py @@ -173,16 +173,28 @@ async def print_children(device): print(f"{name}: {await obj.read()}") +class TSTPandaHDFWriter(PandaHDFWriter): + async def open(self, *args, **kwargs): + desc = await super().open(*args, **kwargs) + # prefix = self._name_provider() + for key in desc: + if "-counter2-out-" in key: + desc[key]["dtype_str"] = " Date: Thu, 21 Mar 2024 16:23:41 -0400 Subject: [PATCH 3/6] Manta async work --- startup/00-startup.py | 10 +-- startup/10-panda.py | 12 ++-- startup/15-manta.py | 148 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 160 insertions(+), 10 deletions(-) create mode 100644 startup/15-manta.py diff --git a/startup/00-startup.py b/startup/00-startup.py index ab5f928..23176f5 100644 --- a/startup/00-startup.py +++ b/startup/00-startup.py @@ -1,11 +1,13 @@ # Make ophyd listen to pyepics. print(f"Loading file {__file__!r} ...") + import datetime import logging import os import subprocess import warnings +import epics import nslsii import ophyd.signal from bluesky.callbacks.broker import post_run, verify_files_saved @@ -30,11 +32,11 @@ mpl=True, epics_context=False, ) -RE.unsubscribe(0) # remove temp databroker subscription +# RE.unsubscribe(0) # remove temp databroker subscription -tiled_client = from_uri("http://localhost:8000", api_key=os.getenv("TILED_API_KEY", "")) -tw = TiledWriter(tiled_client) -RE.subscribe(tw) +# tiled_client = from_uri("http://localhost:8000", api_key=os.getenv("TILED_API_KEY", "")) +# tw = TiledWriter(tiled_client) +# RE.subscribe(tw) # This is needed for ophyd-async to enable 'await <>' instead of 'asyncio.run(<>)': get_ipython().run_line_magic("autoawait", "call_in_bluesky_event_loop") diff --git a/startup/10-panda.py b/startup/10-panda.py index 06386a3..c358557 100644 --- a/startup/10-panda.py +++ b/startup/10-panda.py @@ -185,11 +185,11 @@ async def open(self, *args, **kwargs): return desc -async def instantiate_panda_async(): - async with DeviceCollector(): +def instantiate_panda_async(): + with DeviceCollector(): panda3_async = PandA("XF:31ID1-ES{PANDA:3}:", name="panda3_async") - async with DeviceCollector(): + with DeviceCollector(): dir_prov = UUIDDirectoryProvider(PROPOSAL_DIR) writer3 = TSTPandaHDFWriter( "XF:31ID1-ES{PANDA:3}", @@ -202,7 +202,7 @@ async def instantiate_panda_async(): return panda3_async, writer3 -panda3_async, writer3 = asyncio.run(instantiate_panda_async()) +panda3_async, writer3 = instantiate_panda_async() @AsyncStatus.wrap @@ -243,9 +243,9 @@ def _count_async_panda_run(panda_device, writer): An exception has occurred, use '%tb verbose' to see the full traceback. RuntimeError: asyncio.run() cannot be called from a running event loop """ - asyncio.run(writer.open()) + # asyncio.run(writer.open()) yield from arm(panda_device) - asyncio.run(writer.close()) + # asyncio.run(writer.close()) class TriggerState(str, Enum): diff --git a/startup/15-manta.py b/startup/15-manta.py new file mode 100644 index 0000000..a6e3515 --- /dev/null +++ b/startup/15-manta.py @@ -0,0 +1,148 @@ +print(f"Loading file {__file__!r} ...") + + +import asyncio +from enum import Enum + +from ophyd_async.core import ( + DEFAULT_TIMEOUT, + DetectorControl, + DetectorTrigger, + DetectorWriter, + DeviceCollector, + HardwareTriggeredFlyable, + ShapeProvider, + SignalRW, + SimSignalBackend, + StaticDirectoryProvider, + TriggerInfo, + TriggerLogic, + UUIDDirectoryProvider, + set_sim_value, +) +from ophyd_async.core.async_status import AsyncStatus +from ophyd_async.core.detector import StandardDetector +from ophyd_async.core.device import DeviceCollector +from ophyd_async.epics.areadetector.controllers.vimba_controller import VimbaController +from ophyd_async.epics.areadetector.drivers.vimba_driver import VimbaDriver +from ophyd_async.epics.areadetector.writers.hdf_writer import HDFWriter +from ophyd_async.epics.areadetector.writers.nd_file_hdf import NDFileHDF + +MANTA_PV_PREFIX = "XF:31ID1-ES{GigE-Cam:1}" + + +class TriggerState(str, Enum): + null = "null" + preparing = "preparing" + starting = "starting" + stopping = "stopping" + + +class MantaTriggerLogic(TriggerLogic[int]): + def __init__(self): + self.state = TriggerState.null + + def trigger_info(self, value: int) -> TriggerInfo: + return TriggerInfo( + num=value, trigger=DetectorTrigger.internal, deadtime=2, livetime=2 + ) + + async def prepare(self, value: int): + self.state = TriggerState.preparing + return value + + async def start(self): + self.state = TriggerState.starting + + async def stop(self): + self.state = TriggerState.stopping + + +manta_trigger_logic = MantaTriggerLogic() + + +class MantaShapeProvider(ShapeProvider): + def __init__(self) -> None: + pass + + async def __call__(self): + return (544, 728) + + +def instantiate_panda_async(): + with DeviceCollector(): + manta_async = VimbaDriver(MANTA_PV_PREFIX + "cam1:") + hdf_plugin_manta = NDFileHDF(MANTA_PV_PREFIX + "HDF1:", name="manta_hdf_plugin") + + with DeviceCollector(): + dir_prov = UUIDDirectoryProvider(PROPOSAL_DIR) + manta_writer = HDFWriter( + hdf_plugin_manta, + dir_prov, + lambda: "lab3-manta", + MantaShapeProvider(), + ) + print_children(manta_async) + + return manta_async, manta_writer + + +manta_async, manta_writer = instantiate_panda_async() +manta_controller = VimbaController(manta_async) + +manta_standard_det = StandardDetector( + manta_controller, manta_writer, name="manta_standard_det" +) + + +manta_flyer = HardwareTriggeredFlyable(manta_trigger_logic, [], name="manta_flyer") + + +def manta_stage(): + yield from bps.stage(manta_standard_det) + yield from bps.sleep(5) + + +def manta_fly( + num=10, +): # Note: 724 points are specific for the "rotation_sim_04" panda config! + yield from bps.stage_all(manta_standard_det, manta_flyer) + assert manta_flyer._trigger_logic.state == TriggerState.stopping + yield from bps.prepare(manta_flyer, num, wait=True) + yield from bps.prepare(manta_standard_det, manta_flyer.trigger_info, wait=True) + + detector = manta_standard_det + # detector.controller.disarm.assert_called_once # type: ignore + + yield from bps.open_run() + + yield from bps.kickoff(manta_flyer) + yield from bps.kickoff(detector) + + yield from bps.complete(manta_flyer, wait=True, group="complete") + yield from bps.complete(detector, wait=True, group="complete") + + # Manually incremenet the index as if a frame was taken + # detector.writer.index += 1 + + done = False + while not done: + try: + yield from bps.wait(group="complete", timeout=0.5) + except TimeoutError: + pass + else: + done = True + yield from bps.collect( + manta_standard_det, + stream=True, + return_payload=False, + name="main_stream", + ) + yield from bps.sleep(0.01) + yield from bps.wait(group="complete") + val = yield from bps.rd(manta_writer.hdf.num_captured) + print(f"{val = }") + yield from bps.close_run() + + yield from bps.unstage_all(manta_flyer, manta_standard_det) From 61e0a7b2c853acf0a2b16e58557976eb9350d311 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Thu, 21 Mar 2024 19:01:00 -0400 Subject: [PATCH 4/6] Make manta_async work with RE --- startup/00-startup.py | 20 ++++++++++++-------- startup/15-manta.py | 2 +- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/startup/00-startup.py b/startup/00-startup.py index 23176f5..9d6e5ca 100644 --- a/startup/00-startup.py +++ b/startup/00-startup.py @@ -1,26 +1,27 @@ # Make ophyd listen to pyepics. print(f"Loading file {__file__!r} ...") +import asyncio import datetime import logging import os import subprocess import warnings -import epics +import epicscorelibs.path.pyepics import nslsii import ophyd.signal from bluesky.callbacks.broker import post_run, verify_files_saved from bluesky.callbacks.tiled_writer import TiledWriter -from bluesky.run_engine import call_in_bluesky_event_loop +from bluesky.run_engine import RunEngine, call_in_bluesky_event_loop from databroker.v0 import Broker from IPython import get_ipython from tiled.client import from_uri ophyd.signal.EpicsSignal.set_defaults(connection_timeout=5) # See docstring for nslsii.configure_base() for more details -# this command takes away much of the boilerplate for settting up a profile -# (such as setting up best effort callbacks etc) +# this command takes away much of the boilerplate for setting up a profile +# (such as setting up best-effort callback, etc) nslsii.configure_base( @@ -32,11 +33,14 @@ mpl=True, epics_context=False, ) -# RE.unsubscribe(0) # remove temp databroker subscription -# tiled_client = from_uri("http://localhost:8000", api_key=os.getenv("TILED_API_KEY", "")) -# tw = TiledWriter(tiled_client) -# RE.subscribe(tw) +event_loop = asyncio.get_event_loop() +RE = RunEngine(loop=event_loop) +RE.subscribe(bec) + +tiled_client = from_uri("http://localhost:8000", api_key=os.getenv("TILED_API_KEY", "")) +tw = TiledWriter(tiled_client) +RE.subscribe(tw) # This is needed for ophyd-async to enable 'await <>' instead of 'asyncio.run(<>)': get_ipython().run_line_magic("autoawait", "call_in_bluesky_event_loop") diff --git a/startup/15-manta.py b/startup/15-manta.py index a6e3515..a9fc2b8 100644 --- a/startup/15-manta.py +++ b/startup/15-manta.py @@ -66,7 +66,7 @@ def __init__(self) -> None: pass async def __call__(self): - return (544, 728) + return (544, 728) # y, x def instantiate_panda_async(): From 283d03d76b0eca35fa52f731f490a520c998882a Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Fri, 22 Mar 2024 17:52:16 -0400 Subject: [PATCH 5/6] Manta+Panda fly tomo scan --- startup/05-motors.py | 1 + startup/10-panda.py | 2 +- startup/15-manta.py | 15 +++++- startup/90-plans.py | 121 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 136 insertions(+), 3 deletions(-) diff --git a/startup/05-motors.py b/startup/05-motors.py index abe5dee..d751d54 100644 --- a/startup/05-motors.py +++ b/startup/05-motors.py @@ -6,6 +6,7 @@ class EpicsMotorWithSPMG(EpicsMotor): spmg = Cpt(EpicsSignal, ".SPMG") + velocity = Cpt(EpicsSignal, ".VELO") rot_motor = EpicsMotorWithSPMG("XF:31ID1-OP:1{CMT:1-Ax:X}Mtr", name="rot_motor") diff --git a/startup/10-panda.py b/startup/10-panda.py index c358557..aea7d94 100644 --- a/startup/10-panda.py +++ b/startup/10-panda.py @@ -261,7 +261,7 @@ def __init__(self): def trigger_info(self, value: int) -> TriggerInfo: return TriggerInfo( - num=value, trigger=DetectorTrigger.constant_gate, deadtime=2, livetime=2 + num=value, trigger=DetectorTrigger.constant_gate, deadtime=0.1, livetime=0.1 ) async def prepare(self, value: int): diff --git a/startup/15-manta.py b/startup/15-manta.py index a9fc2b8..d41cebf 100644 --- a/startup/15-manta.py +++ b/startup/15-manta.py @@ -2,6 +2,7 @@ import asyncio +from dataclasses import dataclass from enum import Enum from ophyd_async.core import ( @@ -38,13 +39,22 @@ class TriggerState(str, Enum): stopping = "stopping" +@dataclass +class MantaTriggerSetup: + num_images: int + exposure_time: float + + class MantaTriggerLogic(TriggerLogic[int]): def __init__(self): self.state = TriggerState.null - def trigger_info(self, value: int) -> TriggerInfo: + def trigger_info(self, setup) -> TriggerInfo: return TriggerInfo( - num=value, trigger=DetectorTrigger.internal, deadtime=2, livetime=2 + num=setup.num_images, + trigger=DetectorTrigger.constant_gate, + deadtime=0.1, + livetime=setup.exposure_time, ) async def prepare(self, value: int): @@ -140,6 +150,7 @@ def manta_fly( name="main_stream", ) yield from bps.sleep(0.01) + yield from bps.wait(group="complete") val = yield from bps.rd(manta_writer.hdf.num_captured) print(f"{val = }") diff --git a/startup/90-plans.py b/startup/90-plans.py index d4345ba..396aafc 100644 --- a/startup/90-plans.py +++ b/startup/90-plans.py @@ -102,3 +102,124 @@ def inner(): yield from inner() bps.sleep(0.1) + + +# Number of encoder counts for an entire revolution +COUNTS_PER_REVOLUTION = 8000 +DEG_PER_REVOLUTION = 360 +COUNTS_PER_DEG = COUNTS_PER_REVOLUTION / DEG_PER_REVOLUTION + + +def tomo_demo_async(num_images=21, scan_time=9, start_deg=0, exposure_time=None): + + panda3_pcomp_1 = dict(panda3_async.pcomp.children())["1"] + + step_width_counts = COUNTS_PER_REVOLUTION / (2 * (num_images - 1)) + if int(step_width_counts) != round(step_width_counts, 5): + raise ValueError( + "The number of encoder counts per pulse is not an integer value!" + ) + + # step_time = (scan_time / num_images) + # camera_exposure_time = step_time / 2 + # if exposure_time is not None: + # if exposure_time > step_time: + # raise RuntimeError(f"Your configured exposure time is longer than the step size {step_time}") + camera_exposure_time = exposure_time + + manta_exp_setup = MantaTriggerSetup( + num_images=num_images, exposure_time=camera_exposure_time + ) + + yield from bps.mv( + rot_motor.velocity, 180 / 2 + ) # Make it fast to move to the start position + yield from bps.mv(rot_motor, start_deg - 20) + yield from bps.mv( + rot_motor.velocity, 180 / scan_time + ) # Set the velocity for the scan + start_encoder = start_deg * COUNTS_PER_DEG + + width_in_counts = (180 / scan_time) * COUNTS_PER_DEG * exposure_time + if width_in_counts > step_width_counts: + raise RuntimeError( + f"Your specified exposure time of {exposure_time}s is too long! Calculated width: {width_in_counts}, Step size: {step_width_counts}" + ) + print(f"Exposing camera for {width_in_counts} counts") + + # Set up the pcomp block + yield from bps.mv(panda3_pcomp_1.start, int(start_encoder)) + yield from bps.mv( + panda3_pcomp_1.width, width_in_counts + ) # Width in encoder counts that the pulse will be high + yield from bps.mv(panda3_pcomp_1.step, step_width_counts) + yield from bps.mv(panda3_pcomp_1.pulses, num_images) + + # The setup below is happening in the VimbaController's arm method. + # # Setup camera in trigger mode + # yield from bps.mv(manta_async.trigger_mode, "On") + # yield from bps.mv(manta_async.trigger_source, "Line1") + # yield from bps.mv(manta_async.overlap, "Off") + # yield from bps.mv(manta_async.expose_out_mode, "TriggerWidth") # "Timed" or "TriggerWidth" + + # Stage All! + yield from bps.stage_all(manta_standard_det, manta_flyer) + assert manta_flyer._trigger_logic.state == TriggerState.stopping + yield from bps.prepare(manta_flyer, manta_exp_setup, wait=True) + yield from bps.prepare(manta_standard_det, manta_flyer.trigger_info, wait=True) + + yield from bps.stage_all(panda3_standard_det, panda3_flyer) + assert panda3_flyer._trigger_logic.state == TriggerState.stopping + yield from bps.prepare(panda3_flyer, num_images, wait=True) + yield from bps.prepare(panda3_standard_det, panda3_flyer.trigger_info, wait=True) + + yield from bps.mv(rot_motor, start_deg + DEG_PER_REVOLUTION / 2 + 5) + + detector = panda3_standard_det + # detector.controller.disarm.assert_called_once # type: ignore + + yield from bps.open_run() + + yield from bps.kickoff(panda3_flyer) + yield from bps.kickoff(detector) + + yield from bps.complete(panda3_flyer, wait=True, group="complete") + yield from bps.complete(detector, wait=True, group="complete") + + # Manually incremenet the index as if a frame was taken + # detector.writer.index += 1 + + done = False + while not done: + try: + yield from bps.wait(group="complete", timeout=0.5) + except TimeoutError: + pass + else: + done = True + yield from bps.collect( + detector, + stream=True, + return_payload=False, + name=f"{detector.name}_stream", + ) + yield from bps.collect( + manta_standard_det, + stream=True, + return_payload=False, + name=f"{manta_standard_det.name}_stream", + ) + yield from bps.sleep(0.01) + + yield from bps.wait(group="complete") + yield from bps.close_run() + + panda_val = yield from bps.rd(writer3.hdf.num_captured) + manta_val = yield from bps.rd(manta_writer.hdf.num_captured) + print(f"{panda_val = } {manta_val = }") + + yield from bps.unstage_all(panda3_flyer, detector) + yield from bps.unstage_all(manta_flyer, manta_standard_det) + + # Reset the velocity back to high. + yield from bps.mv(rot_motor.velocity, 180 / 2) From c39a896bb28a6a1afb5f8d4dfe639d20d8180154 Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Tue, 26 Mar 2024 14:31:08 -0400 Subject: [PATCH 6/6] More tweaks of the tomo plan --- startup/15-manta.py | 2 +- startup/90-plans.py | 20 ++++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/startup/15-manta.py b/startup/15-manta.py index d41cebf..cd43462 100644 --- a/startup/15-manta.py +++ b/startup/15-manta.py @@ -52,7 +52,7 @@ def __init__(self): def trigger_info(self, setup) -> TriggerInfo: return TriggerInfo( num=setup.num_images, - trigger=DetectorTrigger.constant_gate, + trigger=DetectorTrigger.edge_trigger, deadtime=0.1, livetime=setup.exposure_time, ) diff --git a/startup/90-plans.py b/startup/90-plans.py index 396aafc..a0c1b06 100644 --- a/startup/90-plans.py +++ b/startup/90-plans.py @@ -120,11 +120,13 @@ def tomo_demo_async(num_images=21, scan_time=9, start_deg=0, exposure_time=None) "The number of encoder counts per pulse is not an integer value!" ) - # step_time = (scan_time / num_images) - # camera_exposure_time = step_time / 2 - # if exposure_time is not None: - # if exposure_time > step_time: - # raise RuntimeError(f"Your configured exposure time is longer than the step size {step_time}") + step_time = scan_time / num_images + camera_exposure_time = step_time / 2 + if exposure_time is not None: + if exposure_time > step_time: + raise RuntimeError( + f"Your configured exposure time is longer than the step size {step_time}" + ) camera_exposure_time = exposure_time manta_exp_setup = MantaTriggerSetup( @@ -149,9 +151,11 @@ def tomo_demo_async(num_images=21, scan_time=9, start_deg=0, exposure_time=None) # Set up the pcomp block yield from bps.mv(panda3_pcomp_1.start, int(start_encoder)) - yield from bps.mv( - panda3_pcomp_1.width, width_in_counts - ) # Width in encoder counts that the pulse will be high + + # Uncomment if using gate trigger mode on camera + # yield from bps.mv( + # panda3_pcomp_1.width, width_in_counts + # ) # Width in encoder counts that the pulse will be high yield from bps.mv(panda3_pcomp_1.step, step_width_counts) yield from bps.mv(panda3_pcomp_1.pulses, num_images)