From 1a0efb9c00efc83a38212f016679b2c0d776b4ab Mon Sep 17 00:00:00 2001 From: Nathan Pemberton Date: Fri, 15 Feb 2019 19:41:18 -0500 Subject: [PATCH 01/12] Remove sudo requirement again (a more permanent fix is still needed) --- marshal | 4 +-- wlutil/install.py | 8 +++++- wlutil/wlutil.py | 66 ++++++++++++++++++++++++----------------------- 3 files changed, 43 insertions(+), 35 deletions(-) diff --git a/marshal b/marshal index 46aec78c..91a913c9 100755 --- a/marshal +++ b/marshal @@ -6,8 +6,8 @@ import logging import wlutil import contextlib -if 'RISCV' not in os.environ: - sys.exit("Please source firesim/sourceme-manager-f1.sh first\n") +# if 'RISCV' not in os.environ: +# sys.exit("Please source firesim/sourceme-manager-f1.sh first\n") # Delete a file but don't throw an exception if it doesn't exist def deleteSafe(pth): diff --git a/wlutil/install.py b/wlutil/install.py index 6649ad3a..ece78889 100644 --- a/wlutil/install.py +++ b/wlutil/install.py @@ -5,7 +5,10 @@ from .wlutil import * # firesim workloads directory -fsWork = (Path(root_dir) / "../../deploy/workloads").resolve() +try: + fsWork = (Path(root_dir) / "../../deploy/workloads").resolve() +except: + fsWork = None readmeTxt="""This workload was generated using firesim-software. See the following config and workload directory for details: @@ -18,6 +21,9 @@ def fullRel(base, target): return os.path.relpath(str(target), start=str(base)) def installWorkload(cfgName, cfgs): + if fsWork == None: + raise RuntimeError("The install command is not supported when firesim-software is checked out standalone (i.e. not a submodule of firesim).") + targetCfg = cfgs[cfgName] # if 'jobs' in targetCfg: # raise NotImplementedError("Jobs not currently supported by the install command") diff --git a/wlutil/wlutil.py b/wlutil/wlutil.py index ff07caae..dd83f7a9 100644 --- a/wlutil/wlutil.py +++ b/wlutil/wlutil.py @@ -112,26 +112,26 @@ def genRunScript(command): # Frustratingly, the same commands don't work/exist on various platforms so we # need to figure out what mounting options are available to us: -# if shutil.which('guestmount') is not None: -# # This is the preferred method because it doesn't require sudo -# @contextmanager -# def mountImg(imgPath, mntPath): -# run(['guestmount', '-a', imgPath, '-m', '/dev/sda', mntPath]) -# try: -# yield mntPath -# finally: -# run(['guestunmount', mntPath]) -# -# elif shutil.whcih('fuse-ext2') is not None: -# # Roughly the same as guestmount -# @contextmanager -# def mountImg(imgPath, mntPath): -# run(['fuse-ext2', '-o', 'rw+', imgPath, mntPath]) -# try: -# yield mntPath -# finally: -# run(['fusermount', '-u', mntPath]) -# +if shutil.which('guestmount') is not None: + # This is the preferred method because it doesn't require sudo + @contextmanager + def mountImg(imgPath, mntPath): + run(['guestmount', '-a', imgPath, '-m', '/dev/sda', mntPath]) + try: + yield mntPath + finally: + run(['guestunmount', mntPath]) + +elif shutil.whcih('fuse-ext2') is not None: + # Roughly the same as guestmount + @contextmanager + def mountImg(imgPath, mntPath): + run(['fuse-ext2', '-o', 'rw+', imgPath, mntPath]) + try: + yield mntPath + finally: + run(['fusermount', '-u', mntPath]) + # elif shutil.which('mount') is not None: # # if True: # # Should be available everywhere, requires sudo @@ -142,17 +142,17 @@ def genRunScript(command): # yield mntPath # finally: # run(['sudo', 'umount', mntPath]) -# -# else: -# raise ImportError("No compatible 'mount' command found") -@contextmanager -def mountImg(imgPath, mntPath): - run(['sudo', 'mount', '-o', 'loop', imgPath, mntPath]) - try: - yield mntPath - finally: - run(['sudo', 'umount', mntPath]) +else: + raise ImportError("No compatible 'mount' command found") + +# @contextmanager +# def mountImg(imgPath, mntPath): +# run(['sudo', 'mount', '-o', 'loop', imgPath, mntPath]) +# try: +# yield mntPath +# finally: +# run(['sudo', 'umount', mntPath]) def toCpio(config, src, dst): log = logging.getLogger() @@ -188,9 +188,11 @@ def copyImgFiles(img, files, direction): # Note: shell=True because f.src is allowed to contain globs # Note: os.path.join can't handle overlay-style concats (e.g. join('foo/bar', '/baz') == '/baz') if direction == 'in': - run('sudo rsync -a --chown=root:root ' + f.src + " " + os.path.normpath(mnt + f.dst), shell=True) + # run('sudo rsync -a --chown=root:root ' + f.src + " " + os.path.normpath(mnt + f.dst), shell=True) + run('rsync -a --chown=root:root ' + f.src + " " + os.path.normpath(mnt + f.dst), shell=True) elif direction == 'out': uid = os.getuid() - run('sudo rsync -a --chown=' + str(uid) + ':' + str(uid) + ' ' + os.path.normpath(mnt + f.src) + " " + f.dst, shell=True) + # run('sudo rsync -a --chown=' + str(uid) + ':' + str(uid) + ' ' + os.path.normpath(mnt + f.src) + " " + f.dst, shell=True) + run('rsync -a --chown=' + str(uid) + ':' + str(uid) + ' ' + os.path.normpath(mnt + f.src) + " " + f.dst, shell=True) else: raise ValueError("direction option must be either 'in' or 'out'") From e86e650f17c63333ef0116e2581519b9a2f45e46 Mon Sep 17 00:00:00 2001 From: Nathan Pemberton Date: Mon, 18 Feb 2019 14:19:11 -0500 Subject: [PATCH 02/12] Remove sudo requirement. Note: this change requires that you install either guestmount or fuse-ext2. --- wlutil/wlutil.py | 36 ++++++++---------------------------- 1 file changed, 8 insertions(+), 28 deletions(-) diff --git a/wlutil/wlutil.py b/wlutil/wlutil.py index dd83f7a9..dd362e78 100644 --- a/wlutil/wlutil.py +++ b/wlutil/wlutil.py @@ -107,9 +107,6 @@ def genRunScript(command): return commandScript -# XXX This isn't working with the initramfs option. Go back to requiring sudo -# for now, I'll revisit later. - # Frustratingly, the same commands don't work/exist on various platforms so we # need to figure out what mounting options are available to us: if shutil.which('guestmount') is not None: @@ -132,34 +129,19 @@ def mountImg(imgPath, mntPath): finally: run(['fusermount', '-u', mntPath]) -# elif shutil.which('mount') is not None: -# # if True: -# # Should be available everywhere, requires sudo -# @contextmanager -# def mountImg(imgPath, mntPath): -# run(['sudo', 'mount', '-o', 'loop', imgPath, mntPath]) -# try: -# yield mntPath -# finally: -# run(['sudo', 'umount', mntPath]) - else: - raise ImportError("No compatible 'mount' command found") - -# @contextmanager -# def mountImg(imgPath, mntPath): -# run(['sudo', 'mount', '-o', 'loop', imgPath, mntPath]) -# try: -# yield mntPath -# finally: -# run(['sudo', 'umount', mntPath]) + # Note: we don't support the 'mount' option because it requires sudo. By + # itself that isn't an issue, but several other commands (like cpio and + # rsync) would also require sudo and that complicates the code too much. + # This feature could be added back if people really want it. + raise ImportError("No compatible 'mount' command found: Please install either guestmount or fuse-ext2") def toCpio(config, src, dst): log = logging.getLogger() with mountImg(src, mnt): # Fedora needs a special init in order to boot from initramfs - run("sudo find -print0 | sudo cpio --owner root:root --null -ov --format=newc > " + dst, shell=True, cwd=mnt) + run("find -print0 | cpio --owner root:root --null -ov --format=newc > " + dst, shell=True, cwd=mnt) # fedora needs a special init to work if config['distro'] == 'fedora': @@ -188,11 +170,9 @@ def copyImgFiles(img, files, direction): # Note: shell=True because f.src is allowed to contain globs # Note: os.path.join can't handle overlay-style concats (e.g. join('foo/bar', '/baz') == '/baz') if direction == 'in': - # run('sudo rsync -a --chown=root:root ' + f.src + " " + os.path.normpath(mnt + f.dst), shell=True) - run('rsync -a --chown=root:root ' + f.src + " " + os.path.normpath(mnt + f.dst), shell=True) + run('sudo rsync -a --chown=root:root ' + f.src + " " + os.path.normpath(mnt + f.dst), shell=True) elif direction == 'out': uid = os.getuid() - # run('sudo rsync -a --chown=' + str(uid) + ':' + str(uid) + ' ' + os.path.normpath(mnt + f.src) + " " + f.dst, shell=True) - run('rsync -a --chown=' + str(uid) + ':' + str(uid) + ' ' + os.path.normpath(mnt + f.src) + " " + f.dst, shell=True) + run('sudo rsync -a --chown=' + str(uid) + ':' + str(uid) + ' ' + os.path.normpath(mnt + f.src) + " " + f.dst, shell=True) else: raise ValueError("direction option must be either 'in' or 'out'") From 264f5c1bcef2e3213234ac8f1ea7dd1359a8e5df Mon Sep 17 00:00:00 2001 From: Nathan Pemberton Date: Mon, 18 Feb 2019 11:42:48 -0800 Subject: [PATCH 03/12] Make fuse-ext2 the default (needed for a* machines). Also remove left-over sudo's that were sneaking around. --- wlutil/wlutil.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/wlutil/wlutil.py b/wlutil/wlutil.py index dd362e78..b409dccf 100644 --- a/wlutil/wlutil.py +++ b/wlutil/wlutil.py @@ -109,25 +109,25 @@ def genRunScript(command): # Frustratingly, the same commands don't work/exist on various platforms so we # need to figure out what mounting options are available to us: -if shutil.which('guestmount') is not None: - # This is the preferred method because it doesn't require sudo +if shutil.which('fuse-ext2') is not None: + # Roughly the same as guestmount @contextmanager def mountImg(imgPath, mntPath): - run(['guestmount', '-a', imgPath, '-m', '/dev/sda', mntPath]) + run(['fuse-ext2', '-o', 'rw+', imgPath, mntPath]) try: yield mntPath finally: - run(['guestunmount', mntPath]) + run(['fusermount', '-u', mntPath]) -elif shutil.whcih('fuse-ext2') is not None: - # Roughly the same as guestmount +elif shutil.which('guestmount') is not None: + # This is the preferred method because it doesn't require sudo @contextmanager def mountImg(imgPath, mntPath): - run(['fuse-ext2', '-o', 'rw+', imgPath, mntPath]) + run(['guestmount', '-a', imgPath, '-m', '/dev/sda', mntPath]) try: yield mntPath finally: - run(['fusermount', '-u', mntPath]) + run(['guestunmount', mntPath]) else: # Note: we don't support the 'mount' option because it requires sudo. By @@ -170,9 +170,9 @@ def copyImgFiles(img, files, direction): # Note: shell=True because f.src is allowed to contain globs # Note: os.path.join can't handle overlay-style concats (e.g. join('foo/bar', '/baz') == '/baz') if direction == 'in': - run('sudo rsync -a --chown=root:root ' + f.src + " " + os.path.normpath(mnt + f.dst), shell=True) + run('rsync -a --chown=root:root ' + f.src + " " + os.path.normpath(mnt + f.dst), shell=True) elif direction == 'out': uid = os.getuid() - run('sudo rsync -a --chown=' + str(uid) + ':' + str(uid) + ' ' + os.path.normpath(mnt + f.src) + " " + f.dst, shell=True) + run('rsync -a --chown=' + str(uid) + ':' + str(uid) + ' ' + os.path.normpath(mnt + f.src) + " " + f.dst, shell=True) else: raise ValueError("direction option must be either 'in' or 'out'") From 7cc604c70d5a8f8a5bbb26b0c42a905348dd22c7 Mon Sep 17 00:00:00 2001 From: Nathan Pemberton Date: Wed, 1 May 2019 20:22:59 -0400 Subject: [PATCH 04/12] pick random tcp port so multiple users can co-exist on one machine --- wlutil/launch.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/wlutil/launch.py b/wlutil/launch.py index d56012bb..5f2a3df2 100644 --- a/wlutil/launch.py +++ b/wlutil/launch.py @@ -1,3 +1,4 @@ +import socket import logging from .wlutil import * @@ -5,6 +6,14 @@ launch_mem = "16384" launch_cores = "4" +# Kinda hacky (technically not guaranteed to give a free port, just very likely) +def get_free_tcp_port(): + tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + tcp.bind(('', 0)) + addr, port = tcp.getsockname() + tcp.close() + return str(port) + # Returns a command string to luanch the given config in spike. Must be called with shell=True. def getSpikeCmd(config, initramfs=False): if 'spike' in config: @@ -23,6 +32,8 @@ def getSpikeCmd(config, initramfs=False): def getQemuCmd(config, initramfs=False): log = logging.getLogger() + launch_port = get_free_tcp_port() + if initramfs: exe = config['bin'] + '-initramfs' else: @@ -37,7 +48,7 @@ def getQemuCmd(config, initramfs=False): '-object', 'rng-random,filename=/dev/urandom,id=rng0', '-device', 'virtio-rng-device,rng=rng0', '-device', 'virtio-net-device,netdev=usernet', - '-netdev', 'user,id=usernet,hostfwd=tcp::10000-:22'] + '-netdev', 'user,id=usernet,hostfwd=tcp::' + launch_port + '-:22'] if 'img' in config and not initramfs: cmd = cmd + ['-device', 'virtio-blk-device,drive=hd0', From 0d01c11c87ab95d4299a53cded2a003708aa3882 Mon Sep 17 00:00:00 2001 From: Nathan Pemberton Date: Mon, 3 Jun 2019 16:52:18 -0400 Subject: [PATCH 05/12] Add initramfs overlay for buildroot (adds /init script). This worked by accident on earlier versions of buildroot, now we do it explicitly. Conflicts: wlutil/wlutil.py --- marshal | 3 --- wlutil/br-initramfs-append.cpio | Bin 0 -> 512 bytes wlutil/wlutil.py | 7 ++++++- 3 files changed, 6 insertions(+), 4 deletions(-) create mode 100644 wlutil/br-initramfs-append.cpio diff --git a/marshal b/marshal index 82f64642..92d5d938 100755 --- a/marshal +++ b/marshal @@ -6,9 +6,6 @@ import logging import wlutil import contextlib -# if 'RISCV' not in os.environ: -# sys.exit("Please source firesim/sourceme-manager-f1.sh first\n") - # Delete a file but don't throw an exception if it doesn't exist def deleteSafe(pth): with contextlib.suppress(FileNotFoundError): diff --git a/wlutil/br-initramfs-append.cpio b/wlutil/br-initramfs-append.cpio new file mode 100644 index 0000000000000000000000000000000000000000..212c31c448134a798f35dba42fd019362e1851ff GIT binary patch literal 512 zcmcJL%MOAt6hyn`D=y&H^a3fcFh)^GOk5d%K%pp!w1yVr=Sv|Zz7}qr%`~TXrVorT zLJ$2g$YK^?r`Su^#xY3a7f1BiDr0fT!k%N|51tM4*fo6@8*wEnO^9pLV0!=cr(maWjdfKoPKa?PPGmDMNvEtu-3^jJf?LW|#d4mdMwnoFVN ug%8Y|e=`3Iqsm;+oC?ll;aG%pZ!`b&n|_^pZ%an7-z_)WbZ6T(`LiD4uWSPV literal 0 HcmV?d00001 diff --git a/wlutil/wlutil.py b/wlutil/wlutil.py index 028580ca..1be1ae20 100644 --- a/wlutil/wlutil.py +++ b/wlutil/wlutil.py @@ -164,9 +164,14 @@ def toCpio(config, src, dst): # Fedora needs a special init in order to boot from initramfs run("find -print0 | cpio --owner root:root --null -ov --format=newc > " + dst, shell=True, cwd=mnt) - # fedora needs a special init to work + # Ideally, the distro's themselves would provide initramfs-based versions. + # However, having two codepaths for disk images and cpio archives + # complicates a bunch of stuff in the rest of marshal. Instead, we maintain + # overlays here that convert a disk-based image to a cpio-based image. if config['distro'] == 'fedora': sp.call("cat " + os.path.join(wlutil_dir, "fedora-initramfs-append.cpio") + " >> " + dst, shell=True) + elif config['distro'] == 'br': + sp.call("cat " + os.path.join(wlutil_dir, "br-initramfs-append.cpio") + " >> " + dst, shell=True) # Apply the overlay directory "overlay" to the filesystem image "img" # Note that all paths must be absolute From a9dfa834b61d43ef77b2a459d2b600d340309826 Mon Sep 17 00:00:00 2001 From: Nathan Pemberton Date: Tue, 4 Jun 2019 20:46:32 -0400 Subject: [PATCH 06/12] Switch to use guestmount exclusively. --- full_test.sh | 34 +++++++++++++++++----------------- wlutil/wlutil.py | 41 ++++++++++------------------------------- 2 files changed, 27 insertions(+), 48 deletions(-) diff --git a/full_test.sh b/full_test.sh index bd0b776e..0855ad5e 100755 --- a/full_test.sh +++ b/full_test.sh @@ -30,23 +30,6 @@ else echo "Success" | tee -a $LOGNAME fi -# Run the specialized tests (tests that are too complicated for ./marshal -# test) -echo "Running clean test" | tee -a $LOGNAME -./test/clean/test.py >> $LOGNAME -if [ ${PIPESTATUS[0]} != 0 ]; then - echo "Failure" | tee -a $LOGNAME - SUITE_PASS=false -fi - -echo "Running incremental test" | tee -a $LOGNAME -./test/incremental/test.py >> $LOGNAME -if [ ${PIPESTATUS[0]} != 0 ]; then - echo "Failure" | tee -a $LOGNAME - SUITE_PASS=false - exit 1 -fi - # Run the bulk tests (all work with the 'test' command) # Note the funny extended globbing, these are just lists of tests that # shouldn't be tested (e.g. we exclude the base configs and some specialized @@ -77,6 +60,23 @@ else echo "Success" | tee -a $LOGNAME fi +# Run the specialized tests (tests that are too complicated for ./marshal +# test) +echo "Running clean test" | tee -a $LOGNAME +./test/clean/test.py >> $LOGNAME +if [ ${PIPESTATUS[0]} != 0 ]; then + echo "Failure" | tee -a $LOGNAME + SUITE_PASS=false +fi + +echo "Running incremental test" | tee -a $LOGNAME +./test/incremental/test.py >> $LOGNAME +if [ ${PIPESTATUS[0]} != 0 ]; then + echo "Failure" | tee -a $LOGNAME + SUITE_PASS=false + exit 1 +fi + echo -e "\n\nMarshal full test complete. Log at: $LOGNAME" if [ $SUITE_PASS = false ]; then echo "FAILURE: Some tests failed" | tee -a $LOGNAME diff --git a/wlutil/wlutil.py b/wlutil/wlutil.py index 1be1ae20..9209950a 100644 --- a/wlutil/wlutil.py +++ b/wlutil/wlutil.py @@ -128,34 +128,13 @@ def genRunScript(command): return commandScript -# Frustratingly, the same commands don't work/exist on various platforms so we -# need to figure out what mounting options are available to us: -if shutil.which('fuse-ext2') is not None: - # Roughly the same as guestmount - @contextmanager - def mountImg(imgPath, mntPath): - run(['fuse-ext2', '-o', 'rw+', imgPath, mntPath]) - try: - yield mntPath - finally: - run(['fusermount', '-u', mntPath]) - -elif shutil.which('guestmount') is not None: - # This is the preferred method because it doesn't require sudo - @contextmanager - def mountImg(imgPath, mntPath): - run(['guestmount', '-a', imgPath, '-m', '/dev/sda', mntPath]) - try: - yield mntPath - finally: - run(['guestunmount', mntPath]) - -else: - # Note: we don't support the 'mount' option because it requires sudo. By - # itself that isn't an issue, but several other commands (like cpio and - # rsync) would also require sudo and that complicates the code too much. - # This feature could be added back if people really want it. - raise ImportError("No compatible 'mount' command found: Please install either guestmount or fuse-ext2") +@contextmanager +def mountImg(imgPath, mntPath): + run(['guestmount', '-a', imgPath, '-m', '/dev/sda', mntPath]) + try: + yield mntPath + finally: + run(['guestunmount', mntPath]) def toCpio(config, src, dst): log = logging.getLogger() @@ -192,13 +171,13 @@ def copyImgFiles(img, files, direction): with mountImg(img, mnt): for f in files: # Overlays may not be owned by root, but the filesystem must be. - # Rsync lets us chown while copying. + # The FUSE mount automatically handles the chown from root to user # Note: shell=True because f.src is allowed to contain globs # Note: os.path.join can't handle overlay-style concats (e.g. join('foo/bar', '/baz') == '/baz') if direction == 'in': - run('rsync -a --chown=root:root ' + f.src + " " + os.path.normpath(mnt + f.dst), shell=True) + run('cp -a ' + f.src + " " + os.path.normpath(mnt + f.dst), shell=True) elif direction == 'out': uid = os.getuid() - run('rsync -a --chown=' + str(uid) + ':' + str(uid) + ' ' + os.path.normpath(mnt + f.src) + " " + f.dst, shell=True) + run('cp -a ' + os.path.normpath(mnt + f.src) + " " + f.dst, shell=True) else: raise ValueError("direction option must be either 'in' or 'out'") From 5b6c40ae93617820764bb985d93c8f46ee41d2d8 Mon Sep 17 00:00:00 2001 From: Nathan Pemberton Date: Tue, 4 Jun 2019 20:50:15 -0400 Subject: [PATCH 07/12] remove sudo requirement from fedora disk conversion --- wlutil/fedora/convert_raw.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wlutil/fedora/convert_raw.sh b/wlutil/fedora/convert_raw.sh index ed5ef3aa..3a482301 100755 --- a/wlutil/fedora/convert_raw.sh +++ b/wlutil/fedora/convert_raw.sh @@ -27,7 +27,7 @@ DD_BS=1048576 # Get the offset of the first partition within the image so we can strip it out # output of parted looks like "###B ####B ###B ..." we cast it to an array here -PART_INFO=(`sudo parted -s $RAWIMG unit B print | tail -2`) +PART_INFO=(`parted -s $RAWIMG unit B print | tail -2`) # The output at index 1 and 2 are the partition start and end byte offset. The : : -1 strips the "B" suffix. # Also convert to units of 1MB blocks (I'd do that above, but parted output is weird) From 83ba20fe257f6f28abf88c06f66ac0d17b80cac1 Mon Sep 17 00:00:00 2001 From: Nathan Pemberton Date: Wed, 5 Jun 2019 20:01:56 +0000 Subject: [PATCH 08/12] Added guestmount to requirements --- centos-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/centos-requirements.txt b/centos-requirements.txt index ad292633..e6786668 100644 --- a/centos-requirements.txt +++ b/centos-requirements.txt @@ -2,3 +2,4 @@ python34 python34-devel python34-pip rsync +libguestfs-tools From 61abc0994c35147108e742add2dac038cb71c760 Mon Sep 17 00:00:00 2001 From: Nathan Pemberton Date: Wed, 12 Jun 2019 16:28:26 -0400 Subject: [PATCH 09/12] Fix race-condition with guestmount (guestmount bug that we need to work around) --- wlutil/wlutil.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/wlutil/wlutil.py b/wlutil/wlutil.py index 9209950a..fe0b56f8 100644 --- a/wlutil/wlutil.py +++ b/wlutil/wlutil.py @@ -7,6 +7,8 @@ import sys import collections import shutil +import psutil +import errno from contextlib import contextmanager wlutil_dir = os.path.normpath(os.path.dirname(__file__)) @@ -128,13 +130,33 @@ def genRunScript(command): return commandScript +# This is like os.waitpid, but it works for non-child processes +def waitpid(pid): + done = False + while not done: + try: + os.kill(pid, 0) + except OSError as err: + if err.errno == errno.ESRCH: + done = True + break + time.sleep(0.25) + @contextmanager def mountImg(imgPath, mntPath): - run(['guestmount', '-a', imgPath, '-m', '/dev/sda', mntPath]) + run(['guestmount', '--pid-file', 'guestmount.pid', '-a', imgPath, '-m', '/dev/sda', mntPath]) try: + with open('./guestmount.pid', 'r') as pidFile: + mntPid = int(pidFile.readline()) yield mntPath finally: run(['guestunmount', mntPath]) + os.remove('./guestmount.pid') + + # There is a race-condition in guestmount where a background task keeps + # modifying the image for a period after unmount. This is the documented + # best-practice (see man guestmount). + waitpid(mntPid) def toCpio(config, src, dst): log = logging.getLogger() From 941efc5f5395a9e6686dcf239a82143755415a6d Mon Sep 17 00:00:00 2001 From: Nathan Pemberton Date: Mon, 24 Jun 2019 22:50:47 +0000 Subject: [PATCH 10/12] Pre-build the base workloads without a timeout so that full_test works on a fresh clone (it would timeout on the first test otherwise due to buildroot). --- full_test.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/full_test.sh b/full_test.sh index 0855ad5e..72b15170 100755 --- a/full_test.sh +++ b/full_test.sh @@ -7,6 +7,11 @@ LOGNAME=$(mktemp results_full_test.XXXX) echo "Running Full Test. Results available in $LOGNAME" +# We pre-build to avoid potential timeouts on a fresh clone +echo "Pre-building base workloads" | tee -a $LOGNAME +./marshal build test/br-base.json +./marshal build test/fedora-base.json + echo "Running launch timeout test (should timeout):" | tee -a $LOGNAME echo "This test will reset your terminal" ./marshal test test/timeout-run.json | grep "timeout while running" From 070739eabd2169b679416bf41097667d938acc1a Mon Sep 17 00:00:00 2001 From: Nathan Pemberton Date: Mon, 24 Jun 2019 23:04:58 +0000 Subject: [PATCH 11/12] Update python package requirements --- centos-requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/centos-requirements.txt b/centos-requirements.txt index e6786668..d44b84e7 100644 --- a/centos-requirements.txt +++ b/centos-requirements.txt @@ -1,5 +1,5 @@ -python34 -python34-devel -python34-pip +python36 +python36-devel +python36-pip rsync libguestfs-tools From 991e34d1971de9e6372577aa7902c7df7a8a9b42 Mon Sep 17 00:00:00 2001 From: Nathan Pemberton Date: Tue, 25 Jun 2019 16:08:03 +0000 Subject: [PATCH 12/12] Add back toolchain check for more obvious errors when you forget to source sourceme.sh --- marshal | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/marshal b/marshal index 92d5d938..33264e77 100755 --- a/marshal +++ b/marshal @@ -5,6 +5,10 @@ import os import logging import wlutil import contextlib +import shutil + +if not shutil.which('riscv64-unknown-linux-gnu-gcc'): + sys.exit("No riscv toolchain detected. Please install riscv-tools.") # Delete a file but don't throw an exception if it doesn't exist def deleteSafe(pth):