Pynq-fud integration (#1120)

* Initial commit for a fud-pynq python script

* Working fud-pyq-script with hardcoded xclbin path

* Finished generalized pynq sript

* Integrate fud-pynq-script into fud stage

Previously, fud-pynq-script accepted a mapping and returned a string,
now ittakes a path to an xclbin file and returns a dictionary conforming
to json standards.

The new execution.py retains all of the previous config settings while
simplifying the kernel execution by running fud-pynq-script.
execution.py is responsible for turning the script's output into a json
file

* Add pynq dependency

* Add pynq dependency

* Fix imports and typing

* Fixed path issues ruining my day

* Made Pynq an optional dependency. removed opencl

* Revert previously introduced fmt changes

* Fixed underfined pynq_script

* Fix import_libs

* make flake happy

Co-authored-by: Nathaniel Navarro <nrn25@cornell.edu>

fud/fud/stages/xilinx/execution.py

@@ -2,12 +2,13 @@
 import os
 import time
 
-import numpy as np
 import simplejson as sjson
 
+
 from fud import errors
 from fud.stages import SourceType, Stage
 from fud.utils import FreshDir, TmpDir
+from pathlib import Path
 
 
 class HwExecutionStage(Stage):
@@ -24,7 +25,7 @@ def __init__(self):
 
     def _define_steps(self, input, builder, config):
         data_path = config["stages", self.name, "data"]
-        
+
         save_temps = bool(config["stages", self.name, "save_temps"])
         waveform = bool(config["stages", self.name, "waveform"])
         if waveform and not save_temps:
@@ -33,14 +34,14 @@ def _define_steps(self, input, builder, config):
                 f"is not. This will generate a WDB file but then immediately "
                 f"delete it. Consider adding `-s {self.name}.save_temps true`."
             )
-        
+
         @builder.step()
         def import_libs():
             """Import optional libraries"""
             try:
-                import pyopencl as cl  # type: ignore
+                from fud.stages.xilinx import fud_pynq_script
 
-                self.cl = cl
+                self.pynq_script = fud_pynq_script
             except ImportError:
                 raise errors.RemoteLibsNotInstalled
 
@@ -50,9 +51,8 @@ def run(xclbin: SourceType.Path) -> SourceType.String:
 
             if data_path is None:
                 raise errors.MissingDynamicConfiguration("fpga.data")
-
-            data = sjson.load(open(data_path), use_decimal=True)
-            xclbin_source = xclbin.open("rb").read()
+            abs_data_path = Path(data_path).resolve()
+            abs_xclbin_path = xclbin.resolve()
 
             # Create a temporary directory with an xrt.ini file that redirects
             # the runtime log to a file so that we can control how it's printed.
@@ -71,8 +71,12 @@ def run(xclbin: SourceType.Path) -> SourceType.String:
                 ]
                 if waveform:
                     xrt_ini_config.append("debug_mode=batch\n")
-                    xrt_ini_config.append(f"user_pre_sim_script={new_dir.name}/pre_sim.tcl\n")
-                    xrt_ini_config.append(f"user_post_sim_script={new_dir.name}/pre_sim.tcl\n")
+                    xrt_ini_config.append(
+                        f"user_pre_sim_script={new_dir.name}/pre_sim.tcl\n"
+                    )
+                    xrt_ini_config.append(
+                        f"user_post_sim_script={new_dir.name}/pre_sim.tcl\n"
+                    )
                 f.writelines(xrt_ini_config)
 
             # Extra Tcl scripts to produce a VCD waveform dump.
@@ -87,49 +91,15 @@ def run(xclbin: SourceType.Path) -> SourceType.String:
                         "close_vcd\n",
                     ])
 
-            ctx = self.cl.create_some_context(0)
-            dev = ctx.devices[0]
-            cmds = self.cl.CommandQueue(ctx, dev)
-            prg = self.cl.Program(ctx, [dev], [xclbin_source])
-
-            prg.build()
-
-            # Work around an intermittent PyOpenCL bug. Using prg.Toplevel
-            # internally accesses prg._source, expecting it to be a normal
-            # attribute instead of a kernel name.
-            kern = self.cl.Kernel(prg, "Toplevel")
-
-            buffers = {}
-            for mem in data.keys():
-                # allocate memory on the device
-                buf = self.cl.Buffer(
-                    ctx,
-                    self.cl.mem_flags.READ_WRITE | self.cl.mem_flags.COPY_HOST_PTR,
-                    # TODO: use real type information
-                    hostbuf=np.array(data[mem]["data"]).astype(np.uint32),
-                )
-                # TODO: use real type information
-                buffers[mem] = buf
-
+            data = sjson.load(open(abs_data_path), use_decimal=True)
             start_time = time.time()
-            #Note that this is the call on v++. This uses global USER_ENV variables
-            #EMCONFIG_PATH=`pwd`
-            #XCL_EMULATION_MODE=hw_emu
-            kern(cmds, (1,), (1,), np.uint32(10000), *buffers.values())
+            # Note that this is the call on v++. This uses global USER_ENV variables
+            # EMCONFIG_PATH=`pwd`
+            # XCL_EMULATION_MODE=hw_emu
+            kernel_output = self.pynq_script.run(abs_xclbin_path, data)
             end_time = time.time()
             log.debug(f"Emulation time: {end_time - start_time} sec")
 
-            # read the result
-            output = {"memories": {}}
-            for name, buf in buffers.items():
-                out_buf = np.zeros_like(data[name]["data"]).astype(np.uint32)
-                self.cl.enqueue_copy(cmds, out_buf, buf)
-                buf.release()
-                output["memories"][name] = list(map(lambda x: int(x), out_buf))
-
-            # cleanup
-            del ctx
-
             # Add xrt log output to our debug output.
             if os.path.exists(xrt_output_logname):
                 log.debug("XRT log:")
@@ -145,9 +115,8 @@ def run(xclbin: SourceType.Path) -> SourceType.String:
                     for line in f.readlines():
                         log.debug(line.strip())
 
-            return sjson.dumps(output, indent=2, use_decimal=True)
+            return sjson.dumps(kernel_output, indent=2, use_decimal=True)
 
         import_libs()
         res = run(input)
         return res
-

fud/fud/stages/xilinx/fud_pynq_script.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+import pynq
+import numpy as np
+from typing import Mapping, Any, Dict
+from pathlib import Path
+from fud.stages.verilator.json_to_dat import parse_fp_widths, float_to_fixed
+from fud.errors import InvalidNumericType
+
+
+# XXX(nathanielnrn): Should xclbin typing only be pathlib.Path, or also accept strings?
+def run(xclbin_path: Path, data: Mapping[str, Any]) -> Dict[str, Any]:
+    """Takes in a json data output and runs pynq using the data provided
+    returns a dictionary that can be converted into json
+
+    `xclbin` is path to relevant xclbin file.
+    Assumes that data is a properly formatted calyx data file.
+    Data file order must match the expected call signature in terms of order
+    Also assume that the data Mapping values type are valid json-type equivalents
+    """
+
+    # pynq.Overlay expects a str
+    # Raises FileNotFoundError if xclbin file does not exist
+    ol = pynq.Overlay(str(xclbin_path.resolve(strict=True)))
+
+    buffers = []
+    for mem in data.keys():
+        ndarray = np.array(data[mem]["data"], dtype=_dtype(mem, data))
+        shape = ndarray.shape
+        buffer = pynq.allocate(shape, dtype=ndarray.dtype)
+        buffer[:] = ndarray[:]
+        buffers.append(buffer)
+
+    for buffer in buffers:
+        buffer.sync_to_device()
+
+    # Equivalent to setting kernel = ol.<Presumably 'Toplevel_1'>
+    kernel = getattr(ol, list(ol.ip_dict)[0])
+    # XXX(nathanielnrn) 2022-07-19: timeout is not currently used anywhere in
+    # generated verilog code, passed in because kernel.xml is generated to
+    # expect it as an argument
+    timeout = 1000
+    kernel.call(timeout, *buffers)
+
+    output = {"memories": {}}
+    # converts needed data from buffers and adds to json output
+    for i, mem in enumerate(data.keys()):
+        buffers[i].sync_from_device()
+        # converts int representation into fixed point
+        if data[mem]["format"]["numeric_type"] == "fixed_point":
+            width, int_width = parse_fp_widths(data[mem]["format"])
+            frac_width = width - int_width
+
+            def convert_to_fp(value: float):
+                float_to_fixed(float(value), frac_width)
+
+            convert_to_fp(buffers[i])
+            output["memories"][mem] = list((buffers[i]))
+        elif data[mem]["format"]["numeric_type"] == "bitnum":
+            output["memories"][mem] = list(map(lambda e: int(e), buffers[i]))
+
+        else:
+            raise InvalidNumericType('Fud only supports "fixed_point" and "bitnum".')
+
+    # PYNQ recommends deleting buffers and freeing overlay
+    del buffers
+    ol.free()
+    return output
+
+
+def _dtype(mem: str, data: Mapping[str, Any]) -> np.dtype:
+    # See https://numpy.org/doc/stable/reference/arrays.dtypes.html for typing
+    # details
+    type_string = "i" if data[mem]["format"]["is_signed"] else "u"
+    byte_size = int(data[mem]["format"]["width"] / 8)
+    type_string = type_string + str(byte_size)
+    return np.dtype(type_string)

fud/pyproject.toml

@@ -28,7 +28,7 @@ remote = [
        "scp"
 ]
 fpga = [
-  "pyopencl==2021.2.8"
+  "pynq"
 ]
 
 [tool.flit.scripts]