From d7ecf5d75512ab59ef097db6d20198d9456d7327 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 6 Jan 2022 07:30:24 -0800 Subject: [PATCH] [fud] Make SSH optional in Xilinx emulation stage (#852) * Start refactoring generic sandbox utility LocalSandbox is a new alternative to RemoteExecution for running commands locally. It's not a drop-in replacement (maybe that should come in the future), but it offers morally equivalent functionality. * Fix some configuration stuff * Improve an error message I don't know why "provide an input file" was suggested, so I took that out. Also, properly interpolate the variable into the example flag. * Document what we know so far * Black formatting * Simplify emulation command execution And also make them work locally, with a utility. * Avoid some hard-coding * Document wdb stage configuration * Fix for name change from #851 --- docs/fud/synthesis.md | 23 ++++ fud/fud/config.py | 1 + fud/fud/errors.py | 5 +- fud/fud/stages/remote_context.py | 68 +++++++++- fud/fud/stages/xilinx/emulation.py | 210 +++++++++-------------------- fud/fud/stages/xilinx/xclbin.py | 59 +------- 6 files changed, 163 insertions(+), 203 deletions(-) diff --git a/docs/fud/synthesis.md b/docs/fud/synthesis.md index c8d24699d9..1eb115dcaf 100644 --- a/docs/fud/synthesis.md +++ b/docs/fud/synthesis.md @@ -83,6 +83,14 @@ The options for `mode` are `hw_emu` (simulation) and `hw` (on-FPGA execution). The device string above is for the [Alveo U50][u50] card, which we have at Cornell, but I honestly don't know how you're supposed to find the right string for a different FPGA target. Hopefully someone will figure this out and document it in the future. +To use hardware emulation, you will also need to configure the `wdb` stage. +It has similar `ssh_host`, `ssh_username`, and `remote` options to the `xclbin` stage. +You will also need to configure the stage to point to your installations of [Vitis][] and [XRT][], like this: + + [stages.wdb] + xilinx_location: /scratch/opt/Xilinx/Vitis/2020.2 + xrt_location: /opt/xilinx/xrt + ### Compile The first step in the Xilinx toolchain is to generate [an `xclbin` executable file][xclbin]. @@ -97,6 +105,20 @@ By default, the Xilinx tools run in a temporary directory that is deleted when ` To instead keep the sandbox directory, use `-s xclbin.save_temps true`. You can then find the results in a directory named `fud-out-N` for some number `N`. +### Emulate + +You can also execute compiled designs through Xilinx hardware emulation. +Use the `wdb` state as your `fud` target: + + fud e -vv foo.xclbin -s wdb.save_temps true -o out.wdb + +This stage produces a Vivado [waveform database (WDB) file][wdb] +Through the magic of `fud`, you can also go all the way from a Calyx program to a `wdb` file in the same way. +There is also a `wdb.save_temps` option, as with the `xclbin` stage. + +You also need to provide a host C++ program via the `wdb.host` parameter, but I don't know much about that yet, so documentation about that will have to wait. +Similarly, I don't yet know what you're supposed to *do* with a WDB file; maybe we should figure out how to produce a VCD instead. + ### How it Works The first step is to generate input files. @@ -131,3 +153,4 @@ This step uses the `v++` tool, with a command line that looks like this: [xclbin]: https://xilinx.github.io/XRT/2021.2/html/formats.html#xclbin [gen_xo]: https://github.com/cucapra/calyx/blob/master/fud/bitstream/gen_xo.tcl [u50]: https://www.xilinx.com/products/boards-and-kits/alveo/u50.html +[wdb]: https://support.xilinx.com/s/article/64000?language=en_US diff --git a/fud/fud/config.py b/fud/fud/config.py index 910692019e..b84d238e27 100644 --- a/fud/fud/config.py +++ b/fud/fud/config.py @@ -79,6 +79,7 @@ "mode": "hw_emu", "ssh_host": "", "ssh_username": "", + "remote": None, "host": None, "save_temps": None, "xilinx_location": "/scratch/opt/Xilinx/Vitis/2020.2", diff --git a/fud/fud/errors.py b/fud/fud/errors.py index 8530646f02..cf1bbf04d1 100644 --- a/fud/fud/errors.py +++ b/fud/fud/errors.py @@ -68,10 +68,9 @@ class MissingDynamicConfiguration(FudError): def __init__(self, variable): msg = ( - "Provide an input file or " - + f"`{variable}' needs to be set. " + f"`{variable}' needs to be set. " + "Use the runtime configuration flag to provide a value: " - + "'-s {variable} '." + + f"'-s {variable} '." ) super().__init__(msg) diff --git a/fud/fud/stages/remote_context.py b/fud/fud/stages/remote_context.py index 0df6afde8b..a2ac46c220 100644 --- a/fud/fud/stages/remote_context.py +++ b/fud/fud/stages/remote_context.py @@ -1,7 +1,9 @@ import logging as log from pathlib import Path from tempfile import NamedTemporaryFile +import shutil +from fud.utils import TmpDir, FreshDir from .. import errors from ..stages import Source, SourceType @@ -150,9 +152,7 @@ def copy_back( remote_tmpdir: SourceType.String, local_tmpdir: SourceType.Directory, ): - """ - Copy files generated on server back to local host. - """ + """Copy files generated on server back to local host.""" with self.SCPClient(client.get_transport()) as scp: scp.get( remote_tmpdir, local_path=f"{local_tmpdir.name}", recursive=True @@ -183,3 +183,65 @@ def fetch_file( local_path = fetch_file(client, remote_tmpdir) self._close(client, remote_tmpdir, keep_tmpdir=keep_tmpdir) return local_path + + +class LocalSandbox: + """A utility for running commands in a temporary directory. + + This is meant as a local alternative to `RemoteExecution`. Like that + utility, this provides steps to create a temporary directory, + execute programs in that temporary directory, and then retrieve + files from it. However, all this happens locally instead of via SSH. + """ + + def __init__(self, stage, save_temps=False): + self.stage = stage + self.save_temps = save_temps + + def create(self, input_files): + """Copy input files to a fresh temporary directory. + + `input_files` is a dict with the same format as `open_and_send`: + it maps local Source paths to destination strings. + + Return a path to the newly-created temporary directory. + """ + + @self.stage.step() + def copy_file( + tmpdir: SourceType.String, + src_path: SourceType.Path, + dest_path: SourceType.String, + ): + """Copy an input file.""" + shutil.copyfile(src_path, Path(tmpdir) / dest_path) + + tmpdir = Source( + FreshDir() if self.save_temps else TmpDir(), + SourceType.Directory, + ) + for src_path, dest_path in input_files.items(): + if not isinstance(src_path, Source): + src_path = Source(src_path, SourceType.Path) + if not isinstance(dest_path, Source): + dest_path = Source(dest_path, SourceType.String) + copy_file(tmpdir, src_path, dest_path) + + self.tmpdir = tmpdir + return tmpdir + + def get_file(self, name): + """Retrieve a file from the sandbox directory.""" + + @self.stage.step() + def read_file( + tmpdir: SourceType.Directory, + name: SourceType.String, + ) -> SourceType.Path: + """Read an output file.""" + return Path(tmpdir.name) / name + + return read_file( + self.tmpdir, + Source(name, SourceType.String), + ) diff --git a/fud/fud/stages/xilinx/emulation.py b/fud/fud/stages/xilinx/emulation.py index 02f180cbed..eb4128cfb9 100644 --- a/fud/fud/stages/xilinx/emulation.py +++ b/fud/fud/stages/xilinx/emulation.py @@ -2,9 +2,10 @@ from pathlib import Path -from fud.stages import Stage, SourceType -from fud.utils import TmpDir +from fud.stages import Stage, SourceType, Source from fud import errors +from fud.stages.remote_context import RemoteExecution, LocalSandbox +from fud.utils import shell class HwEmulationStage(Stage): @@ -17,16 +18,18 @@ def __init__(self, config): input_type=SourceType.Path, output_type=SourceType.Path, config=config, - description="Runs Vivado hw emulation", + description="Runs Vivado hardware emulation", ) - xilinx_location = self.config["stages", self.name, "xilinx_location"] - xrt_location = self.config["stages", self.name, "xrt_location"] + self.xilinx_location = self.config["stages", self.name, "xilinx_location"] + self.xrt_location = self.config["stages", self.name, "xrt_location"] self.setup_commands = ( - f"source {xilinx_location}/settings64.sh && source {xrt_location}/setup.sh" + f"source {self.xilinx_location}/settings64.sh && " + f"source {self.xrt_location}/setup.sh" ) self.host_cpp = self.config["stages", self.name, "host"] + self.save_temps = bool(self.config["stages", self.name, "save_temps"]) self.xrt = ( Path(self.config["global", "futil_directory"]) @@ -41,30 +44,27 @@ def __init__(self, config): / "sim_script.tcl" ) self.mode = self.config["stages", self.name, "mode"] - self.device = "xilinx_u50_gen3x16_xdma_201920_3" + self.device = "xilinx_u50_gen3x16_xdma_201920_3" # TODO: Hard-coded. # remote execution - self.SSHClient = None - self.SCPClient = None - self.ssh_host = self.config["stages", self.name, "ssh_host"] - self.ssh_user = self.config["stages", self.name, "ssh_username"] + self.remote_exec = RemoteExecution(self) self.temp_location = self.config["stages", "xclbin", "temp_location"] self.setup() - def _define_steps(self, input_data): - @self.step() - def import_libs(): - """Import remote libs.""" - try: - from paramiko import SSHClient - from scp import SCPClient + def _shell(self, client, cmd): + """Run a command, either locally or remotely.""" + if self.remote_exec.use_ssh: + _, stdout, stderr = client.exec_command(cmd) + for chunk in iter(lambda: stdout.readline(2048), ""): + log.debug(chunk.strip()) + log.debug(stderr.read().decode("UTF-8").strip()) - self.SSHClient = SSHClient - self.SCPClient = SCPClient - except ModuleNotFoundError: - raise errors.RemoteLibsNotInstalled + else: + stdout = shell(cmd, capture_stdout=False) + log.debug(stdout) + def _define_steps(self, input_data): @self.step() def check_host_cpp(): """ @@ -73,151 +73,75 @@ def check_host_cpp(): if self.host_cpp is None: raise errors.MissingDynamicConfiguration("wdb.host") - @self.step() - def establish_connection() -> SourceType.UnTyped: - """ - Establish SSH connection - """ - client = self.SSHClient() - client.load_system_host_keys() - client.connect(self.ssh_host, username=self.ssh_user) - return client - - @self.step() - def make_remote_tmpdir(client: SourceType.UnTyped) -> SourceType.String: - """ - Execution `mktemp -d` on server. - """ - _, stdout, _ = client.exec_command(f"mktemp -d -p {self.temp_location}") - return stdout.read().decode("ascii").strip() - - @self.step() - def send_files( - client: SourceType.UnTyped, - tmpdir: SourceType.String, - xclbin: SourceType.Path, - ): - """ - Copy files over ssh channel - """ - with self.SCPClient(client.get_transport()) as scp: - scp.put(xclbin, remote_path=f"{tmpdir}/kernel.xclbin") - scp.put(self.host_cpp, remote_path=f"{tmpdir}/host.cpp") - scp.put(self.xrt, remote_path=f"{tmpdir}/xrt.ini") - scp.put(self.sim_script, remote_path=f"{tmpdir}/sim_script.tcl") - - @self.step() - def setup_environment(client: SourceType.UnTyped): - """ - Source Xilinx scripts - """ - @self.step() def compile_host(client: SourceType.UnTyped, tmpdir: SourceType.String): """ Compile the host code """ - _, stdout, stderr = client.exec_command( - " ".join( - [ - f"cd {tmpdir}", - "&&", - "g++", - "-I/opt/xilinx/xrt/include", - "-I/scratch/opt/Xilinx/Vivado/2020.2/include", - "-Wall -O0 -g -std=c++14 -fmessage-length=0", - "host.cpp", - "-o 'host'", - "-L/opt/xilinx/xrt/lib -lOpenCL -lpthread -lrt -lstdc++", - ] - ) + cmd = ( + f"cd {tmpdir} && " + "g++ " + f"-I{self.xrt_location}/include " + f"-I{self.xilinx_location}/include " + "-Wall -O0 -g -std=c++14 -fmessage-length=0 " + "host.cpp " + "-o 'host' " + f"-L{self.xrt_location}/lib -lOpenCL -lpthread -lrt -lstdc++" ) - - for chunk in iter(lambda: stdout.readline(2048), ""): - log.debug(chunk.strip()) - log.debug(stderr.read().decode("UTF-8").strip()) + self._shell(client, cmd) @self.step() def generate_emconfig(client: SourceType.UnTyped, tmpdir: SourceType.String): """ Generate emconfig.json """ - _, stdout, stderr = client.exec_command( - " ".join( - [ - f"cd {tmpdir}", - "&&", - "/scratch/opt/Xilinx/Vitis/2020.2/bin/emconfigutil", - f"--platform {self.device}", - "--od .", - ] - ) + cmd = ( + f"cd {tmpdir} && " + f"{self.xilinx_location}/bin/emconfigutil " + f"--platform {self.device} " + "--od ." ) - - for chunk in iter(lambda: stdout.readline(2048), ""): - log.debug(chunk.strip()) - log.debug(stderr.read().decode("UTF-8").strip()) + self._shell(client, cmd) @self.step() def emulate(client: SourceType.UnTyped, tmpdir: SourceType.String): """ Emulation the xclbin """ - _, stdout, stderr = client.exec_command( - " ".join( - [ - f"cd {tmpdir}", - "&&", - self.setup_commands, - "&&", - f"XCL_EMULATION_MODE={self.mode}", - "./host", - "kernel.xclbin", - self.device, - ] - ) + cmd = ( + f"cd {tmpdir} && {self.setup_commands} && " + f"XCL_EMULATION_MODE={self.mode} " + f"./host kernel.xclbin {self.device}" ) + self._shell(client, cmd) - for chunk in iter(lambda: stdout.readline(2048), ""): - log.debug(chunk.strip()) - log.debug(stderr.read().decode("UTF-8").strip()) - - @self.step() - def download_wdb( - client: SourceType.UnTyped, - tmpdir: SourceType.String, - ) -> SourceType.Stream: - """ - Download xclbin file - """ - local_tmpdir = TmpDir() - wdb_path = Path(local_tmpdir.name) / "kernel.wdb" - with self.SCPClient(client.get_transport()) as scp: - scp.get( - f"{tmpdir}/xilinx_u50_gen3x16_xdma_201920_3-0-kernel.wdb", - local_path=str(wdb_path), - ) - return wdb_path.open("rb") + check_host_cpp() - @self.step() - def cleanup(client: SourceType.UnTyped, tmpdir: SourceType.String): - """ - Close SSH Connection and cleanup temporaries. - """ - if self.config["stages", self.name, "save_temps"] is None: - client.exec_command("rm -r {tmpdir}") - else: - print(tmpdir) - client.close() + file_map = { + input_data: "kernel.xclbin", + self.host_cpp: "host.cpp", + self.xrt: "xrt.ini", + self.sim_script: "sim_script.tcl", + } + if self.remote_exec.use_ssh: + self.remote_exec.import_libs() + client, tmpdir = self.remote_exec.open_and_send(file_map) + else: + sandbox = LocalSandbox(self, self.save_temps) + tmpdir = sandbox.create(file_map) + client = Source(None, SourceType.UnTyped) - check_host_cpp() - client = establish_connection() - tmpdir = make_remote_tmpdir(client) - send_files(client, tmpdir, input_data) compile_host(client, tmpdir) generate_emconfig(client, tmpdir) emulate(client, tmpdir) - wdb = download_wdb(client, tmpdir) - cleanup(client, tmpdir) - return wdb + wdb_name = f"{self.device}-0-kernel.wdb" + if self.remote_exec.use_ssh: + return self.remote_exec.close_and_get( + client, + tmpdir, + wdb_name, + keep_tmpdir=self.save_temps, + ) + else: + return sandbox.get_file(wdb_name) diff --git a/fud/fud/stages/xilinx/xclbin.py b/fud/fud/stages/xilinx/xclbin.py index 22a04aa472..383b9fc191 100644 --- a/fud/fud/stages/xilinx/xclbin.py +++ b/fud/fud/stages/xilinx/xclbin.py @@ -1,11 +1,10 @@ import logging as log from pathlib import Path -import shutil from fud.stages import Source, SourceType, Stage -from fud.stages.remote_context import RemoteExecution +from fud.stages.remote_context import RemoteExecution, LocalSandbox from fud.stages.futil import FutilStage -from fud.utils import TmpDir, FreshDir, shell +from fud.utils import shell class XilinxStage(Stage): @@ -61,45 +60,7 @@ def _shell(self, client, cmd): stdout = shell(cmd, capture_stdout=False) log.debug(stdout) - def _sandbox(self, input_files): - """Copy input files to a fresh temporary directory. - - `input_files` is a dict with the same format as `open_and_send`: - it maps local Source paths to destination strings. - - Return a path to the newly-created temporary directory. - """ - - @self.step() - def copy_file( - tmpdir: SourceType.String, - src_path: SourceType.Path, - dest_path: SourceType.String, - ): - """Copy an input file.""" - shutil.copyfile(src_path, Path(tmpdir) / dest_path) - - tmpdir = Source( - FreshDir() if self.save_temps else TmpDir(), - SourceType.Directory, - ) - for src_path, dest_path in input_files.items(): - if not isinstance(src_path, Source): - src_path = Source(src_path, SourceType.Path) - if not isinstance(dest_path, Source): - dest_path = Source(dest_path, SourceType.String) - copy_file(tmpdir, src_path, dest_path) - return tmpdir - def _define_steps(self, input_data): - # Step 1: Make a new temporary directory - @self.step() - def mktmp() -> SourceType.Directory: - """ - Make temporary directory to store generated files. - """ - return TmpDir() - # Step 2: Compile input using `-b xilinx` @self.step() def compile_xilinx(inp: SourceType.Stream) -> SourceType.Path: @@ -176,14 +137,6 @@ def compile_xclbin(client: SourceType.UnTyped, tmpdir: SourceType.String): ) self._shell(client, cmd) - @self.step() - def read_file( - tmpdir: SourceType.Directory, - name: SourceType.String, - ) -> SourceType.Path: - """Read an output file.""" - return Path(tmpdir.name) / name - if self.remote_exec.use_ssh: self.remote_exec.import_libs() @@ -200,7 +153,8 @@ def read_file( if self.remote_exec.use_ssh: client, tmpdir = self.remote_exec.open_and_send(file_map) else: - tmpdir = self._sandbox(file_map) + sandbox = LocalSandbox(self, self.save_temps) + tmpdir = sandbox.create(file_map) client = Source(None, SourceType.UnTyped) package_xo(client, tmpdir) @@ -214,7 +168,4 @@ def read_file( keep_tmpdir=self.save_temps, ) else: - return read_file( - tmpdir, - Source("xclbin/kernel.xclbin", SourceType.String), - ) + return sandbox.get_file("xclbin/kernel.xclbin")