TEST: swupd support and testing

.gitmodules

@@ -40,3 +40,6 @@
 [submodule "iot-web-layers"]
 	path = iot-web-layers
 	url = https://github.com/intel/iot-web-layers.git
+[submodule "meta-swupd"]
+	path = meta-swupd
+	url = https://github.com/pohly/meta-swupd.git

meta-refkit-core/bbappends/meta-intel-realsense/recipes-support/librealsense/librealsense_1.12.1.bbappend

@@ -0,0 +1,4 @@
+# History of the git repo was rewritten so that the SRCREV is no longer on the master
+# branch...
+SRC_URI_remove_df-refkit-config = "git://github.com/IntelRealSense/librealsense.git;branch=master"
+SRC_URI_prepend_df-refkit-config = "git://github.com/IntelRealSense/librealsense.git;nobranch=1 "

meta-refkit-core/classes/image-dsk.bbclass

@@ -70,3 +70,7 @@ do_uefiapp_deploy_append () {
 }
 
 do_uefiapp_deploy[depends] += "rmc-db:do_deploy"
+
+# temporary fix, should be in meta-intel/classes/uefi-comboapp.bbclass
+# Patch submitted: [meta-intel][PATCH] uefi-comboapp.bbclass: install files under pseudo
+do_uefiapp_deploy[fakeroot] = "1"

meta-refkit-core/classes/refkit-image.bbclass

@@ -190,6 +190,11 @@ FEATURE_PACKAGES_tools-debug_append = " valgrind"
 FEATURE_PACKAGES_computervision = "packagegroup-computervision"
 FEATURE_PACKAGES_computervision-test = "packagegroup-computervision-test"
 
+# If we use the UEFI combo app and do swupd-based image updates, then
+# we also need to hook into the post-update systemd hook to update
+# the combo app in the EFI partition.
+FEATURE_PACKAGES_swupd = "${@ 'efi-combo-trigger' if oe.types.boolean(d.getVar('REFKIT_USE_DSK_IMAGES') or '0') else '' }"
+
 LICENSE = "MIT"
 
 # See local.conf.sample for explanations.
@@ -378,6 +383,10 @@ DEPENDS += "${@ 'attr-native' if '${REFKIT_IMAGE_STRIP_SMACK}' else '' }"
 # made due to filesystem metadata time stamps being in future.
 APPEND_append = " fsck.mode=skip"
 
+# Do not mount read/write in the initramfs when the goal is to have a read-only
+# rootfs. Not sure why OE-core does not do that itself.
+APPEND_append = "${@ bb.utils.contains('IMAGE_FEATURES', 'read-only-rootfs', ' ro', '', d)} "
+
 # Ensure that images preserve Smack labels and IMA/EVM.
 inherit ${@bb.utils.contains_any('IMAGE_FEATURES', ['ima','smack'], 'xattr-images', '', d)}
 

meta-refkit-core/classes/systemd-sysusers.bbclass

@@ -80,4 +80,4 @@ ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('DISTRO_FEATURES', 'systemd'
 # available in OE-core. However, that code is still not suitable
 # (https://bugzilla.yoctoproject.org/show_bug.cgi?id=9789) and thus we
 # have to use our own version.
-ROOTFS_POSTPROCESS_COMMAND_remove = "systemd_create_users"
+ROOTFS_POSTPROCESS_COMMAND_remove = "systemd_create_users systemd_create_users;"

meta-refkit-core/conf/distro/include/stateless.inc

@@ -46,9 +46,12 @@ STATELESS_RM_pn-systemd += " \
 # systemd/src/core/machine-id-setup.c).
 STATELESS_ETC_WHITELIST += "machine-id"
 
-# These files must be ignored by swupd.
+# These files must be ignored by swupd, unless the root file system
+# is read-only, in which case they do not actually get modified in the
+# partition. Using swupd in such a setup only works when doing A/B
+# partitioning.
 STATEFUL_FILES += "/etc/machine-id"
-SWUPD_FILE_BLACKLIST_append = " ${STATEFUL_FILES}"
+SWUPD_FILE_BLACKLIST_append = "${@ bb.utils.contains('IMAGE_FEATURES', 'read-only-rootfs', '', ' ' + d.getVar('STATEFUL_FILES'), d) }"
 
 # Depend on the installed components and thus has to be computed on
 # the device. Handled by systemd during booting or updates.

meta-refkit-core/lib/oeqa/selftest/cases/refkit_ostree.py

@@ -1,15 +1,8 @@
-from oeqa.selftest.systemupdate.systemupdatebase import SystemUpdateBase
+from oeqa.selftest.systemupdate.httpupdate import HTTPUpdate
 
-from oeqa.utils.commands import runqemu, get_bb_vars, bitbake
-
-import errno
-import http.server
 import os
-import stat
-import tempfile
-import threading
 
-class RefkitOSTreeUpdateBase(SystemUpdateBase):
+class RefkitOSTreeUpdateBase(HTTPUpdate):
     """
     System update tests for refkit-image-common using OSTree.
     """
@@ -21,52 +14,11 @@ class RefkitOSTreeUpdateBase(SystemUpdateBase):
     IMAGE_BBAPPEND = IMAGE_PN + '.bbappend'
     IMAGE_BBAPPEND_UPDATE = IMAGE_BBAPPEND
 
-    # Address and port of OSTree HTTPD inside the virtual machine's
-    # slirp network.
-    OSTREE_SERVER = '10.0.2.100:8080'
-
-    # Global variables are the same for all recipes,
-    # but RECIPE_SYSROOT_NATIVE is specific to socat-native.
-    BB_VARS = get_bb_vars([
-        'DEPLOY_DIR',
-        'MACHINE',
-        'RECIPE_SYSROOT_NATIVE',
-        ],
-                          'socat-native')
-
-    def track_for_cleanup(self, name):
-        """
-        Run a single test with NO_CLEANUP=<anything> oe-selftest to not clean up after the test.
-        """
-        if 'NO_CLEANUP' not in os.environ:
-            super().track_for_cleanup(name)
-
-    def boot_image(self, overrides):
-        # We don't know the final port yet, so instead we create a placeholder script
-        # for qemu to use and rewrite that script once we are ready. The kernel refuses
-        # to execute a shell script while we have it open, so here we close it
-        # and clean up ourselves.
-        #
-        # The helper script also keeps command line handling a bit simpler (no whitespace
-        # in -netdev parameter), which may or may not be relevant.
-        self.ostree_netcat = tempfile.NamedTemporaryFile(mode='w', prefix='ostree-netcat-', dir=os.getcwd(), delete=False)
-        self.ostree_netcat.close()
-        os.chmod(self.ostree_netcat.name, stat.S_IRUSR|stat.S_IWUSR|stat.S_IXUSR)
-        self.track_for_cleanup(self.ostree_netcat.name)
-
-        qemuboot_conf = os.path.join(self.image_dir_test,
-                                     '%s-%s.qemuboot.conf' % (self.IMAGE_PN, self.BB_VARS['MACHINE']))
-        with open(qemuboot_conf) as f:
-            conf = f.read()
-        with open(qemuboot_conf, 'w') as f:
-            f.write('\n'.join([x for x in conf.splitlines() if not x.startswith('qb_slirp_opt')]))
-            f.write('\nqb_slirp_opt = -netdev user,id=net0,guestfwd=tcp:%s-cmd:%s\n' % \
-                    (self.OSTREE_SERVER, self.ostree_netcat.name))
-        return runqemu(self.IMAGE_PN,
-                       discard_writes=False, ssh=False,
-                       overrides=overrides,
-                       runqemuparams='ovmf slirp nographic',
-                       image_fstype='wic')
+    def setUp(self):
+        # We cannot get the actual OSTREE_REPO for the
+        # image here, so we just assume that it is in the usual place.
+        self.REPO_DIR = os.path.join(HTTPUpdate.BB_VARS['DEPLOY_DIR'], 'ostree-repo')
+        super().setUp()
 
     def stop_update_service(self, qemu):
         cmd = '''systemctl stop refkit-update.service'''
@@ -76,72 +28,28 @@ def stop_update_service(self, qemu):
         return True
 
     def update_image(self, qemu):
-        # We need to bring up some simple HTTP server for the
-        # OSTree repo. We cannot get the actual OSTREE_REPO for the
-        # image here, so we just assume that it is in the usual place.
-        # For the sake of simplicity we change into that directory
-        # because then we can use SimpleHTTPRequestHandler.
-        ostree_repo = os.path.join(self.BB_VARS['DEPLOY_DIR'], 'ostree-repo')
-        old_cwd = os.getcwd()
-        server = None
-        try:
-            # We need to stop the refkit-udpate systemd service before starting
-            # the HTTP server (and thus making any update available) to prevent
-            # the service from racing with us and potentially winning, doing a
-            # full update cycle including a final reboot.
-            self.stop_update_service(qemu)
-
-            os.chdir(ostree_repo)
-            class OSTreeHTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
-                def log_message(s, format, *args):
-                    msg = format % args
-                    self.logger.info(msg)
-
-            handler = OSTreeHTTPRequestHandler
-
-            def create_httpd():
-                for port in range(9999, 10000):
-                    try:
-                        server = http.server.HTTPServer(('localhost', port), handler)
-                        return server
-                    except OSError as ex:
-                        if ex.errno != errno.EADDRINUSE:
-                            raise
-                self.fail('no port available for OSTree HTTP server')
-
-            server = create_httpd()
-            port = server.server_port
-            self.logger.info('serving OSTree repo %s on port %d' % (ostree_repo, port))
-            helper = threading.Thread(name='OSTree HTTPD', target=server.serve_forever)
-            helper.start()
-            # netcat can't be assumed to be present. Build and use socat instead.
-            # It's a bit more complicated but has the advantage that it is in OE-core.
-            socat = os.path.join(self.BB_VARS['RECIPE_SYSROOT_NATIVE'], 'usr', 'bin', 'socat')
-            if not os.path.exists(socat):
-                bitbake('socat-native:do_addto_recipe_sysroot', output_log=self.logger)
-            self.assertExists(socat, 'socat-native was not built as expected')
-            with open(self.ostree_netcat.name, 'w') as f:
-                f.write('''#!/bin/sh
-exec %s 2>>/tmp/ostree.log -D -v -d -d -d -d STDIO TCP:localhost:%d
-''' % (socat, port))
+        # We need to stop the refkit-udpate systemd service before starting
+        # the HTTP server (and thus making any update available) to prevent
+        # the service from racing with us and potentially winning, doing a
+        # full update cycle including a final reboot.
+        self.stop_update_service(qemu)
+
+        return super().update_image(qemu)
+
+    def update_image_via_http(self, qemu):
+        # Use the updater, refkit-ostree-update, in a one-shot mode
+        # attempting just a single update cycle for the test case.
+        # Also override the post-apply hook to only run the UEFI app
+        # update hook. It is a bit of a hack but we don't want the rest
+        # of the hooks run, especially not the reboot hook, to avoid
+        # prematurely rebooting the qemu instance and this is the easiest
+        # way to achieve just that for now.
+        cmd = '''ostree config set 'remote "updates".url' http://%s && refkit-ostree-update --one-shot --post-apply-hook /usr/share/refkit-ostree/hooks/post-apply.d/00-update-uefi-app''' % self.HTTPD_SERVER
+        status, output = qemu.run_serial(cmd, timeout=600)
+        self.assertEqual(1, status, 'Failed to run command "%s":\n%s' % (cmd, output))
+        self.logger.info('Successful (?) update with %s:\n%s' % (cmd, output))
+        return True
 
-            # Use the updater, refkit-ostree-update, in a one-shot mode
-            # attempting just a single update cycle for the test case.
-            # Also override the post-apply hook to only run the UEFI app
-            # update hook. It is a bit of a hack but we don't want the rest
-            # of the hooks run, especially not the reboot hook, to avoid
-            # prematurely rebooting the qemu instance and this is the easiest
-            # way to achieve just that for now.
-            cmd = '''ostree config set 'remote "updates".url' http://%s && refkit-ostree-update --one-shot --post-apply-hook /usr/share/refkit-ostree/hooks/post-apply.d/00-update-uefi-app''' % self.OSTREE_SERVER
-            status, output = qemu.run_serial(cmd, timeout=600)
-            self.assertEqual(1, status, 'Failed to run command "%s":\n%s' % (cmd, output))
-            self.logger.info('Successful (?) update:\n%s' % output)
-            return True
-        finally:
-            os.chdir(old_cwd)
-            if server:
-                server.shutdown()
-                server.server_close()
 
 class RefkitOSTreeUpdateTestAll(RefkitOSTreeUpdateBase):
     def test_update_all(self):
@@ -176,16 +84,3 @@ class RefkitOSTreeUpdateTestDev(RefkitOSTreeUpdateTestAll, metaclass=RefkitOSTre
 
     IMAGE_PN_UPDATE = 'refkit-image-update-ostree-modified'
     IMAGE_BBAPPEND_UPDATE = IMAGE_PN_UPDATE + '.bbappend'
-
-    def setUpLocal(self):
-        super().setUpLocal()
-        def create_image_bb(pn):
-            bb = pn + '.bb'
-            self.track_for_cleanup(bb)
-            self.append_config('BBFILES_append = " %s"' % os.path.abspath(bb))
-            with open(bb, 'w') as f:
-                f.write('require ${META_REFKIT_CORE_BASE}/recipes-images/images/refkit-image-common.bb\n')
-                f.write('OSTREE_BRANCHNAME = "${DISTRO}/${MACHINE}/%s"\n' % self.IMAGE_PN)
-                f.write('''IMAGE_FEATURES_append = "${@ bb.utils.filter('DISTRO_FEATURES', 'stateless', d)}"\n''')
-        create_image_bb(self.IMAGE_PN)
-        create_image_bb(self.IMAGE_PN_UPDATE)