stateless support

meta-refkit-core/classes/refkit-image.bbclass

@@ -80,8 +80,9 @@ IMAGE_FEATURES[validitems] += " \
 # building without swupd), or by defining additional bundles via
 # SWUPD_BUNDLES.
 IMAGE_FEATURES += " \
-    ${@bb.utils.contains('DISTRO_FEATURES', 'ima', 'ima', '', d)} \
-    ${@bb.utils.contains('DISTRO_FEATURES', 'smack', 'smack', '', d)} \
+    ${@ bb.utils.filter('DISTRO_FEATURES', 'ima', d) } \
+    ${@ bb.utils.filter('DISTRO_FEATURES', 'smack', d) } \
+    ${@ bb.utils.filter('DISTRO_FEATURES', 'stateless', d) } \
     ${@ 'muted' if (d.getVar('IMAGE_MODE') or 'production') == 'production' else 'autologin' } \
     ${REFKIT_IMAGE_EXTRA_FEATURES} \
 "

meta-refkit-core/classes/refkit-sanity.bbclass

@@ -46,9 +46,10 @@ python refkit_qa_image () {
             if os.path.islink(path):
                 target = os.readlink(path)
                 final_target = resolve_links(target, root)
-                if not os.path.exists(final_target) and not final_target[len(rootfs):] in whitelist:
-                    bb.error("Dangling symlink: %s -> %s -> %s does not resolve to a valid filesystem entry." %
-                             (path, target, final_target))
+                local_target = final_target[len(rootfs):]
+                if not os.path.exists(final_target) and not local_target in whitelist:
+                    bb.error("Dangling symlink: %s -> %s -> %s (= %s) does not resolve to a valid filesystem entry and %s not in REFKIT_QA_IMAGE_SYMLINK_WHITELIST." %
+                             (path, target, local_target, final_target, local_target))
                     qa_sane = False
 
     if not qa_sane:

meta-refkit-core/classes/systemd-sysusers.bbclass

@@ -70,6 +70,14 @@ systemd_sysusers_create () {
                     ;;
                 esac
             done
+
+            # We are done with the file, it's not needed anymore.
+            # This is also a workaround for systemd creating a "nobody"
+            # group for the "u nobody" entry in basic.conf.
+            # The code above doesn't do that because there is a "nobody"
+            # user already in /etc/passwd. Probably the base configuration
+            # should have a similar group (https://bugzilla.yoctoproject.org/show_bug.cgi?id=11766).
+            rm "$conf"
         fi
     done
 }

meta-refkit-core/conf/distro/include/refkit-config.inc

@@ -58,6 +58,15 @@ REFKIT_DEFAULT_DISTRO_FEATURES += "refkit-config"
 # Enable OSTree system update support.
 REFKIT_DEFAULT_DISTRO_FEATURES += "ostree"
 
+# Reconfigure and/or patch some recipes to support "stateless" images
+# better (stateless = configuration in /etc can be created locally or
+# isn't needed at all). Note that the actual changes are defined by
+# the stateless*.inc files included by a distro config like
+# refkit.conf, i.e. merely including refkit-config.inc does not
+# have much effect even when "stateless" is enabled as distro feature.
+REFKIT_DEFAULT_DISTRO_FEATURES += "stateless"
+require conf/distro/include/stateless.inc
+
 # Remove currently unsupported distro features from global defaults
 REFKIT_DEFAULT_DISTRO_FEATURES_REMOVE += "x11 3g"
 

meta-refkit-core/conf/distro/include/stateless-factory.inc

@@ -0,0 +1,155 @@
+# This include file is a proof-of-concept for using systemd's
+# tmpfiles.d mechanism to populate /etc from factory defaults.
+#
+# This approach has several drawbacks:
+# - When /etc gets populated this way, the resulting files
+#   look to a system update mechanism like OSTree and swupd
+#   like they were created locally by an administor, with
+#   the result that they won't be touched during a system
+#   update even when the factory defaults change.
+# - It only works for files which are not needed by
+#   by systemd itself during the early boot phase, otherwise
+#   the device will be in a different state after the initial
+#   boot compared to the state after the first reboot.
+#   For example, /etc/hostname gets moved into factory
+#   defaults here, but then only becomes effective after
+#   a reboot.
+#
+# Overall it is better to remove the need for files in /etc entirely
+# (difficult, often needs patches) or use a system update mechanism
+# which has at least some support for /etc (like OSTree, or swupd
+# when built in non-stateless mode).
+
+# thermald: upstart init file not needed,
+# config file can be in factory.
+# TODO: remove the file during packaging
+STATELESS_MV_ROOTFS += " \
+    init/thermald.conf=factory \
+"
+STATELESS_MV_ROOTFS += " \
+    thermald=factory \
+"
+
+# Moving /etc/hostname has the effect that a reboot is required
+# before the configured hostname becomes effective again. As not
+# much depends on it, that seems a reasonable default.
+STATELESS_MV_ROOTFS += " \
+    hostname=factory \
+"
+
+# By ensuring that udevd starts after tmpfiles, we can move
+# its main config file into the factory defaults.
+STATELESS_MV_ROOTFS += " \
+    udev/udev.conf=factory:systemd-udevd.service \
+"
+
+# Move away ld.so.conf and let systemd's factory reset mechanism re-create
+# it during boot. For this to work reliably, ldconfig.service must run
+# after systemd-tmpfiles-setup.service. Normally they run in parallel.
+STATELESS_MV_ROOTFS += " \
+    ld.so.conf=factory:ldconfig.service \
+"
+
+# systemd/system.conf and systemd/journald.conf can be moved to
+# /usr/share/doc if and only if they only contains the default,
+# commented out values, because non-default values must already be set
+# before these daemons start.
+#
+# For journald.conf that is problematic, because systemd_232.bb
+# changes journald.conf instead of compiling systemd with different
+# defaults. That could be changed. For now we ignore those
+# modifications and thus accept that the first boot without
+# journald.conf will not run quite as it would normally.
+#
+# Service settings can be moved to /usr because they are part
+# of the system.
+#
+# All remaining systemd config files may or may not have been
+# modified and thus get treated as factory defaults.
+STATELESS_POSTPROCESS += " stateless_mv_systemd_conf;"
+stateless_mv_systemd_conf () {
+    for config in system.conf journald.conf; do
+        if [ -e ${IMAGE_ROOTFS}${sysconfdir}/systemd/$config ]; then
+            if settings=`grep -v -e '^\[.*\]$' -e '^#' -e '^$' -e '^RuntimeMaxUse=' -e '^ForwardToSyslog=' ${IMAGE_ROOTFS}/etc/systemd/$config`; then
+                bbfatal "stateless: ${IMAGE_ROOTFS}/etc/systemd/$config contains more than just comments, cannot remove:\n$settings"
+            fi
+            mkdir -p ${IMAGE_ROOTFS}${datadir}/doc/etc/systemd
+            mv ${IMAGE_ROOTFS}${sysconfdir}/systemd/$config ${IMAGE_ROOTFS}${datadir}/doc/etc/systemd
+        fi
+    done
+}
+STATELESS_MV_ROOTFS += " \
+    systemd/system=${systemd_system_unitdir} \
+    xdg/systemd=factory \
+    systemd=factory \
+"
+
+# Several files in /etc/ssl can become factory defaults.
+# /etc/ssl/certs and /etc/ssl itself will be dealt with below.
+STATELESS_MV_ROOTFS += " \
+    ssl/openssl.cnf=factory \
+    ssl/openssl.cnf.real=factory \
+    ssl/private=factory \
+"
+
+# We could just dump /etc/ssl/certs entirely into the factory
+# defaults, but that sounds redundant, because the content
+# is already generated from read-only system content. Instead,
+# we extend systemd-tmpfiles-setup.service so that it
+# also runs update-ca-certificates.
+STATELESS_POSTPROCESS += " stateless_rm_etc_ssl_certs;"
+stateless_rm_etc_ssl_certs () {
+    if [ -e ${IMAGE_ROOTFS}${sbindir}/update-ca-certificates ] &&
+       [ -e ${IMAGE_ROOTFS}${systemd_system_unitdir}/systemd-tmpfiles-setup.service ]; then
+        echo "ExecStartPost=/bin/sh -c '[ -e ${sysconfdir}/ssl/certs/ca-certificates.crt ] || ${sbindir}/update-ca-certificates'" >>${IMAGE_ROOTFS}${systemd_system_unitdir}/systemd-tmpfiles-setup.service
+        rm -rf ${IMAGE_ROOTFS}${sysconfdir}/ssl/certs
+        # If empty now, /etc/ssl can be removed, too.
+        if rmdir ${IMAGE_ROOTFS}${sysconfdir}/ssl; then
+            echo "d ${sysconfdir}/ssl 0755 root root - -" >>${IMAGE_ROOTFS}${libdir}/tmpfiles.d/stateless.conf
+        fi
+        echo "d ${sysconfdir}/ssl/certs 0755 root root - -" >>${IMAGE_ROOTFS}${libdir}/tmpfiles.d/stateless.conf
+    fi
+}
+
+# OE chooses the actual resolver (for example, ConnMan vs systemd) at
+# build time by symlinking /etc/resolv.conf to the actual .conf file.
+# We need to preserve that choice.
+STATELESS_MV_ROOTFS += " \
+    resolv.conf=factory \
+"
+
+# Various things that systemd and journald do not need when they
+# start.
+STATELESS_MV_ROOTFS += " \
+    asound.conf=factory \
+    bluetooth=factory \
+    busybox.links.nosuid=factory \
+    busybox.links.suid=factory \
+    ca-certificates.conf=factory \
+    dbus-1=factory \
+    default=factory \
+    environment=factory \
+    filesystems=factory \
+    grub.d=factory \
+    host.conf=factory \
+    inputrc=factory \
+    issue=factory \
+    issue.net=factory \
+    libnl=factory \
+    mke2fs.conf=factory \
+    motd=factory \
+    network=factory \
+    nftables=factory \
+    os-release=factory \
+    profile=factory \
+    request-key.conf=factory \
+    resolv-conf.connman=factory \
+    resolv-conf.systemd=factory \
+    security=factory \
+    ssh=factory \
+    skel=factory \
+    timestamp=factory \
+    udhcpc.d=factory \
+    version=factory \
+    wpa_supplicant.conf=factory \
+"

meta-refkit-core/conf/distro/include/stateless-login.inc

@@ -0,0 +1,54 @@
+# Moves login-related files from /etc to /usr/share/doc/etc. Relies
+# on sane defaults in the binaries which read these files. Don't use
+# this include file if the distro actually depends on non-standard
+# settings in those files!
+#
+# BEWARE: depends on non-upstream patch, therefore not currently
+# used in IoT Refkit and might not even compile.
+
+STATELESS_MV_ROOTFS += " \
+    login.defs \
+    securetty \
+    shells \
+"
+
+# Enable logins without /etc/login.defs.
+STATELESS_SRC_append_pn-shadow = " \
+    https://raw.githubusercontent.com/clearlinux-pkgs/shadow/2aae81d2f493e340f454e6888c79f71c0414726c/0001-Do-not-bail-out-on-missing-login.defs.patch \
+    7bf3f3df680fe1515deca2e7bc1715759616f101156650c95172366a79817662 \
+    https://raw.githubusercontent.com/clearlinux-pkgs/shadow/2aae81d2f493e340f454e6888c79f71c0414726c/stateless-login.patch \
+    3bb9bc5936111fac2cfd9723423281c98533740c5ca564152488d9ba33021cc5 \
+"
+
+# Use /usr/share/pam.d instead of /usr/lib/pam.d (for the sake of consistency?)
+# and move /etc files to it. We must prevent systemd from re-creating the files
+# from its own builtin copies.
+STATELESS_SRC_append_pn-libpam = " \
+    https://raw.githubusercontent.com/clearlinux-pkgs/Linux-PAM/b71399c80514afa9411b00aef2be721338a77893/0001-libpam-Keep-existing-pamdir-for-transition.patch \
+    25761101f785878dc7817344f484f670de5723df2eccc17dad9236af446cb890 \
+"
+STATELESS_MV_ROOTFS += "pam.d=${datadir}/pam.d"
+STATELESS_POSTPROCESS += " stateless_rm_systemd_pamd_factory;"
+stateless_rm_systemd_pamd_factory () {
+    rm -rf ${IMAGE_ROOTFS}${datadir}/factory/etc/pam.d
+    if [ -f ${IMAGE_ROOTFS}${libdir}/tmpfiles.d/etc.conf ]; then
+        sed -i -e 's;^\(C */etc/pam.d *.*\);# stateless: \1;' \
+            ${IMAGE_ROOTFS}${libdir}/tmpfiles.d/etc.conf
+    fi
+}
+
+# Allow logins without /etc/login.defs, /etc/securetty or /etc/shells.
+STATELESS_SRC_append_pn-libpam = " \
+    https://raw.githubusercontent.com/clearlinux-pkgs/Linux-PAM/0681d308b660919e6a7ee71be41397dbc8516519/0003-pam_env-Only-report-non-ENOENT-errors-for-env-file.patch \
+    5b6866931e70524ed29cc2b2f5abf31f732658441207d441ec00cbcb9f04833e \
+    https://raw.githubusercontent.com/clearlinux-pkgs/Linux-PAM/aa0bf6295ec8faa96cad1094806a545aae03247e/0004-pam_shells-Support-a-stateless-configuration-by-defa.patch \
+    35d3ca298728aab229b1b82e01ae6b7d0f7be11b0e71c7d18d92ebc8069087aa \
+"
+
+# TODO (?): avoid log entry about "Couldn't open /etc/securetty" each time
+# pam_securetty is used. Written for libpam 1.2.1, does not apply to 1.3.0
+# because the code was modified. Not particularly important as pam_securetty
+# seems unused in OE-core.
+#SRC_URI_append_pn-libpam = " \
+#    https://raw.githubusercontent.com/clearlinux-pkgs/Linux-PAM/0681d308b660919e6a7ee71be41397dbc8516519/0001-pam_securetty-Do-not-report-non-fatal-documented-beh.patch \
+#"

meta-refkit-core/conf/distro/include/stateless-nss-altfiles.inc

@@ -0,0 +1,67 @@
+# Install nss-altfiles, activate it in nsswitch.conf, and move the
+# STATELESS_ALTFILES (like passwd) from /etc into
+# /usr/lib/defaults/etc.
+#
+# As we do this after other ROOTFS_POSTPROCESS_COMMAND,
+# a default root password makes it into the read-only defaults
+# which - if done - is probably intended for debug images.
+#
+# BEWARE: depends on non-upstream patches, therefore not currently
+# used in IoT Refkit and might not even compile.
+
+STATELESS_EXTRA_INSTALL += "nss-altfiles"
+STATELESS_POSTPROCESS += " stateless_activate_altfiles;"
+
+stateless_activate_altfiles () {
+    # This adds "altfiles" as fallback after "compat" or "files".
+    # Relies on nsswitch.conf, which may have been moved already
+    # by stateless_activate_nsswitch.
+    nsswitch_conf=${IMAGE_ROOTFS}/${sysconfdir}/nsswitch.conf
+    if ! [ -e $nsswitch_conf ]; then
+        nsswitch_conf=${IMAGE_ROOTFS}${datadir}/defaults/etc/nsswitch.conf
+        if ! [ -e $nsswitch_conf ]; then
+            bbfatal "nsswitch.conf neither in ${IMAGE_ROOTFS}/${sysconfdir} nor in ${IMAGE_ROOTFS}${datadir}/defaults/etc, require for nss-altfiles."
+        fi
+    fi
+    install -d ${IMAGE_ROOTFS}${datadir}/defaults/etc
+    sed -i -e 's/files/files altfiles/' -e 's/compat/compat altfiles/' $nsswitch_conf
+}
+
+STATELESS_ALTFILES = "hosts services protocols rpc passwd group shadow gshadow"
+STATELESS_MV_ROOTFS += " \
+    ${@ ' '.join('%s=${datadir}/defaults/etc/%s' % (x,x) for x in '${STATELESS_ALTFILES}'.split())} \
+"
+
+# Do not bail out in "adduser" when /etc/passwd is missing.
+STATELESS_SRC_append_pn-busybox = " \
+    file://adduser-enable-use-without-etc-passwd.patch None \
+"
+
+# Teach shadow about altfiles in /usr/defaults/etc and /usr/defaults/skel.
+# For example, setting a password will copy an existing entry from there into /etc.
+STATELESS_SRC_append_pn-shadow = " \
+    https://raw.githubusercontent.com/clearlinux-pkgs/shadow/2aae81d2f493e340f454e6888c79f71c0414726c/0003-Do-not-fail-on-missing-files-in-etc-create-them-inst.patch \
+    3df4182a48a60dc796a2472812adc1a96146c461e6951646c4baaf47e80ed943 \
+    https://raw.githubusercontent.com/clearlinux-pkgs/shadow/2aae81d2f493e340f454e6888c79f71c0414726c/0004-Force-use-shadow-even-if-missing.patch \
+    8e744ae7779b64d7d9668dc2e9bbf42840dd4ed668b66c6bc22bd88837914bd5 \
+    https://raw.githubusercontent.com/clearlinux-pkgs/shadow/2aae81d2f493e340f454e6888c79f71c0414726c/0005-Create-dbs-with-correct-permissions.patch \
+    cb669ad9e99fba3672733524d4e8671b69a86d303f02d915580fc8af586c2aef \
+    https://raw.githubusercontent.com/clearlinux-pkgs/shadow/2aae81d2f493e340f454e6888c79f71c0414726c/0006-Make-usermod-read-altfiles.patch \
+    618e1c6b80f03143c614c9338284cae7928b8fed0a726eed6d8b6f38fdb3d5e5 \
+    https://raw.githubusercontent.com/clearlinux-pkgs/shadow/2aae81d2f493e340f454e6888c79f71c0414726c/stateless-adduser.patch \
+    8fff0b1c52712050b3652d26c8a5faf2acc4cf458964c04a6ca1d28d1d928f2e \
+    https://raw.githubusercontent.com/clearlinux-pkgs/shadow/6d0c85ab07e6c7dd399953f3b9fc24947f910bc8/stateless-gpasswd.patch \
+    e79a3fac817240ebe3144bab67e7ab5f1247b28b59310a13aa9f2cca33d20451 \
+    https://raw.githubusercontent.com/clearlinux-pkgs/shadow/2aae81d2f493e340f454e6888c79f71c0414726c/stateless-useradd.patch \
+    6f47bd7c5df44a1c4dab1bd102c5a8f0f60cf40fd5c6b4c1afd6f7758f280162 \
+    https://raw.githubusercontent.com/clearlinux-pkgs/shadow/d34359528e24569457b8ee8f66d6f2991a291c67/stateless-usermod.patch \
+    af825f9c02834eb7ec34f3ef4c1db0dbc2aed985d02e1c3bc6e8deba5f4ebf68 \
+"
+
+# Required for setting root password when /etc is empty, because
+# otherwise PAM's "is changing the password allowed" check fails,
+# leading to a "permission denied" error before the password prompt.
+STATELESS_SRC_append_pn-libpam = " \
+    https://raw.githubusercontent.com/clearlinux-pkgs/Linux-PAM/b71399c80514afa9411b00aef2be721338a77893/0002-Support-altfiles-locations.patch \
+    53636e3e68a60cef4012735d881cffbd3e653b104e55d94d05826c48b8ec9830 \
+"

meta-refkit-core/conf/distro/include/stateless-nsswitch.inc

@@ -0,0 +1,27 @@
+# Enables the use of /etc/nsswitch.conf as local override for
+# the defaults in /usr/defaults/etc/nsswitch.conf.
+#
+# BEWARE: depends on non-upstream patch, therefore not currently
+# used in IoT Refkit and might not even compile.
+
+# Beware that creating an /etc/nsswitch.conf without actual entries
+# causes a segfault. Reported upstream:
+# https://lists.clearlinux.org/pipermail/dev/2017-July/000927.html
+STATELESS_SRC_append_pn-glibc = " \
+    https://raw.githubusercontent.com/clearlinux-pkgs/glibc/e54b638ef6b5f838e99f1f055474ef2603dfce19/nsswitch-altfiles.patch \
+    82b66bc66d935aed845ae51d0ea7188dbc964ae17bda715f7114805ef5cc915d \
+"
+stateless_activate_nsswitch () {
+    # nsswitch.conf gets moved to /usr and is not needed anymore
+    # in /etc (see stateless_glibc_altfiles_patch).
+    install -d ${IMAGE_ROOTFS}${datadir}/defaults/etc
+    mv ${IMAGE_ROOTFS}/${sysconfdir}/nsswitch.conf ${IMAGE_ROOTFS}${datadir}/defaults/etc/nsswitch.conf
+
+    # We must not let systemd re-create it during boot with the systemd defaults.
+    if [ -f ${IMAGE_ROOTFS}${libdir}/tmpfiles.d/etc.conf ]; then
+        sed -i -e 's;^\(C */etc/nsswitch.conf *.*\);# stateless: \1;' \
+            ${IMAGE_ROOTFS}${libdir}/tmpfiles.d/etc.conf
+    fi
+    rm -f ${IMAGE_ROOTFS}${datadir}/factory/etc/nsswitch.conf
+}
+STATELESS_POSTPROCESS += " stateless_activate_nsswitch;"