diff --git a/meta-refkit-core/classes/refkit-image.bbclass b/meta-refkit-core/classes/refkit-image.bbclass index 7ae98794dc..edcdf5b59a 100644 --- a/meta-refkit-core/classes/refkit-image.bbclass +++ b/meta-refkit-core/classes/refkit-image.bbclass @@ -80,8 +80,9 @@ IMAGE_FEATURES[validitems] += " \ # building without swupd), or by defining additional bundles via # SWUPD_BUNDLES. IMAGE_FEATURES += " \ - ${@bb.utils.contains('DISTRO_FEATURES', 'ima', 'ima', '', d)} \ - ${@bb.utils.contains('DISTRO_FEATURES', 'smack', 'smack', '', d)} \ + ${@ bb.utils.filter('DISTRO_FEATURES', 'ima', d) } \ + ${@ bb.utils.filter('DISTRO_FEATURES', 'smack', d) } \ + ${@ bb.utils.filter('DISTRO_FEATURES', 'stateless', d) } \ ${@ 'muted' if (d.getVar('IMAGE_MODE') or 'production') == 'production' else 'autologin' } \ ${REFKIT_IMAGE_EXTRA_FEATURES} \ " diff --git a/meta-refkit-core/classes/refkit-sanity.bbclass b/meta-refkit-core/classes/refkit-sanity.bbclass index 907e01a04d..f2d23f8d71 100644 --- a/meta-refkit-core/classes/refkit-sanity.bbclass +++ b/meta-refkit-core/classes/refkit-sanity.bbclass @@ -46,9 +46,10 @@ python refkit_qa_image () { if os.path.islink(path): target = os.readlink(path) final_target = resolve_links(target, root) - if not os.path.exists(final_target) and not final_target[len(rootfs):] in whitelist: - bb.error("Dangling symlink: %s -> %s -> %s does not resolve to a valid filesystem entry." % - (path, target, final_target)) + local_target = final_target[len(rootfs):] + if not os.path.exists(final_target) and not local_target in whitelist: + bb.error("Dangling symlink: %s -> %s -> %s (= %s) does not resolve to a valid filesystem entry and %s not in REFKIT_QA_IMAGE_SYMLINK_WHITELIST." % + (path, target, local_target, final_target, local_target)) qa_sane = False if not qa_sane: diff --git a/meta-refkit-core/classes/stateless.bbclass b/meta-refkit-core/classes/stateless.bbclass index 8424506252..80839eed2e 100644 --- a/meta-refkit-core/classes/stateless.bbclass +++ b/meta-refkit-core/classes/stateless.bbclass @@ -1,52 +1,165 @@ -# This moves files out of /etc. It gets applied both -# to individual packages (to avoid or at least catch problems -# early) as well as the entire rootfs (to catch files not -# contained in packages). +# This moves files out of /etc. It gets applied during +# rootfs creation, so packages do not need to be modified +# (although configuring them differently may lead to +# better results). -# Package QA check which greps for known bad paths which should -# not be used anymore, like files which used to be in /etc and -# got moved elsewhere. -STATELESS_DEPRECATED_PATHS ??= "" +# Images are made stateless when "stateless" is in IMAGE_FEATURES. +# By default, that feature is off because it is uncertain which +# images need and support it. +# IMAGE_FEATURES_append_pn-my-stateless-image = " stateless" -# Check not activated by default, can be done in distro with: -# ERROR_QA += "stateless" - -# If set to True, a recipe gets configured with -# sysconfdir=${datadir}/defaults. If set to a path, that -# path is used instead. In both cases, /etc typically gets -# ignored and the component no longer can be configured by -# the device admin. -STATELESS_RELOCATE ??= "False" - -# A space-separated list of recipes which may contain files in /etc. -STATELESS_PN_WHITELIST ??= "" +# There's a QA check in do_rootfs that warns or errors out when /etc +# is not empty in a stateless image. Because /etc does not actually +# need to be empty (for example, when using OSTree), that check is off +# by default. Valid values: no/warn/error +STATELESS_ETC_CHECK_EMPTY ?= "no" # A space-separated list of shell patterns. Anything matching a -# pattern is allowed in /etc. Changing this influences the QA check in -# do_package and do_rootfs. -STATELESS_ETC_WHITELIST ??= "${STATELESS_ETC_DIR_WHITELIST}" +# pattern is allowed in /etc. Changing this influences the QA check. +STATELESS_ETC_WHITELIST ??= "" -# A subset of STATELESS_ETC_WHITELIST which also influences do_install -# and determines which directories to keep. +# Determines which directories to keep in /etc although they are +# empty. Normally such directories get removed. Influences the +# QA check and the actual rootfs mangling. STATELESS_ETC_DIR_WHITELIST ??= "" # A space-separated list of entries in /etc which need to be moved # away. Default is to move into ${datadir}/doc/${PN}/etc. The actual # new name can also be given with old-name=new-name, as in # "pam.d=${datadir}/pam.d". -STATELESS_MV ??= "" +# +# "factory" as special target name moves the item under +# /usr/share/factory/etc and adds it to +# /usr/lib/tmpfiles.d/stateless.conf, so systemd will re-recreate +# when missing. This runs after journald has been started and local +# filesystems are mounted, so things required by those operations +# cannot use the factory mechanism. +# +# Gets applied before the normal ROOTFS_POSTPROCESS_COMMANDs. +STATELESS_MV_ROOTFS ??= "" # A space-separated list of entries in /etc which can be removed # entirely. -STATELESS_RM ??= "" - -# Same as the previous ones, except that they get applied to the rootfs -# before running ROOTFS_POSTPROCESS_COMMANDs. STATELESS_RM_ROOTFS ??= "" -STATELESS_MV_ROOTFS ??= "" + +# Semicolon-separated commands which get run after the normal +# ROOTFS_POSTPROCESS_COMMAND, if the image is meant to be stateless. +STATELESS_POSTPROCESS ??= "" + +# Extra packages to be installed into stateless images. +STATELESS_EXTRA_INSTALL ??= "" + +# STATELESS_SRC can be used to inject source code or patches into +# SRC_URI of a recipe if (and only if) the 'stateless' distro feature is set. +# It is a list of pairs. +# +# This is similar to: +# SRC_URI_pn-foo = "http://some.example.com/foo.patch;name=foo" +# SRC_URI[foo.sha256sum] = "1234" +# +# Setting the hash sum in SRC_URI has the drawback of namespace +# collisions and triggering a world rebuilds for each varflag change, +# because SRC_URI is modified for all recipes (in contrast to +# normal variables, there's no syntax for setting varflags +# per recipe). STATELESS_SRC avoids that because it gets expanded +# seperately for each recipe. +# +# STATELESS_SRC is useful as an alternative for creating .bbappend +# files. Long-term, all patches included this way should become part +# of the upstream layers and then stateless.bbclass also no longer +# needs to be inherited globally. +STATELESS_SRC = "" + ########################################################################### +python () { + import urllib + import os + import string + src = bb.utils.contains('DISTRO_FEATURES', 'stateless', d.getVar('STATELESS_SRC').split(), [], d) + while src: + url = src.pop(0) + if not src: + bb.fatal('STATELESS_SRC must contain pairs of url + shasum') + shasum = src.pop(0) + name = os.path.basename(urllib.parse.urlparse(url).path) + name = ''.join(filter(lambda x: x in string.ascii_letters, name)) + d.appendVar('SRC_URI', ' %s;name=%s' % (url, name)) + d.setVarFlag('SRC_URI', '%s.sha256sum' % name, shasum) +} + +# "stateless" IMAGE_FEATURES definition +IMAGE_FEATURES[validitems] += "stateless" +FEATURE_PACKAGES_stateless = "${STATELESS_EXTRA_INSTALL}" + +# Several post-install scripts modify /etc. +# For example: +# /etc/shells - gets extended when installing a shell package +# /etc/passwd - adduser in postinst extends it +# /etc/systemd/system - has several .wants entries +# +# Instead of completely changing how OE configures images, +# stateless images just take those potentially modified /etc entries +# and makes them part of the read-only system. + +# This can be done in different ways: +# 1. permanently move them into /usr and ensure that software looks +# for entries under both /etc and /usr (example: nss-altfiles +# for a read-only system user and group database) +# 2. move files in /etc to /usr/share/doc/etc and do not restore +# them during booting in those cases where a) the file mirrors +# the builtin defaults of the component using them and b) the +# component works without the file present. +# 3. use a system update and boot mechanism which creates /etc from +# system defaults before booting (example: OSTree) +# 4. restore files in /etc during the early boot phase (example: +# systemd tmpfiles.d) +# +# Case 2 is hard to do in a post-process step, because it's impossible +# to know whether the file in /etc represents builtin defaults. While +# stateless.bbclass has support for this, it's something that is better +# done as part of component packaging. +# +# In case 3 and 4, modifying /etc is possible, but then future system +# updates of the modified files will be ignored. +# +ROOTFS_POSTUNINSTALL_COMMAND_append = "${@ bb.utils.contains('IMAGE_FEATURES', 'stateless', ' stateless_mangle_rootfs;', '', d) }" + +python stateless_mangle_rootfs () { + from oe.utils import execute_pre_post_process + cmds = d.getVar('STATELESS_POSTPROCESS') + execute_pre_post_process(d, cmds) + + rootfsdir = d.getVar('IMAGE_ROOTFS', True) + docdir = rootfsdir + d.getVar('datadir', True) + '/doc/etc' + whitelist = (d.getVar('STATELESS_ETC_WHITELIST', True) or '').split() + dirwhitelist = (d.getVar('STATELESS_ETC_DIR_WHITELIST', True) or '').split() + stateless_mangle(d, rootfsdir, docdir, + (d.getVar('STATELESS_MV_ROOTFS', True) or '').split(), + (d.getVar('STATELESS_RM_ROOTFS', True) or '').split(), + dirwhitelist) + import os + etcdir = os.path.join(rootfsdir, 'etc') + valid = True + etc_empty = d.getVar('STATELESS_ETC_CHECK_EMPTY') + etc_empty_allowed = ('no', 'warn', 'error') + if etc_empty not in etc_empty_allowed: + bb.fatal('STATELESS_ETC_CHECK_EMPTY = "%s" not one of the valid choices (%s)' % + (etc_empty, '/'.join(etc_empty_allowed))) + if etc_empty != 'no': + for dirpath, dirnames, filenames in os.walk(etcdir): + for entry in filenames + [x for x in dirnames if os.path.islink(x)]: + fullpath = os.path.join(dirpath, entry) + etcentry = fullpath[len(etcdir) + 1:] + if not stateless_is_whitelisted(etcentry, whitelist) and \ + not stateless_is_whitelisted(etcentry, dirwhitelist): + bb.warn('stateless: rootfs contains %s' % fullpath) + valid = False + if not valid and etc_empty == 'error': + bb.fatal('stateless: /etc not empty') +} + def stateless_is_whitelisted(etcentry, whitelist): import fnmatch for pattern in whitelist: @@ -54,11 +167,14 @@ def stateless_is_whitelisted(etcentry, whitelist): return True return False -def stateless_mangle(d, root, docdir, stateless_mv, stateless_rm, dirwhitelist, is_package): +def stateless_mangle(d, root, docdir, stateless_mv, stateless_rm, dirwhitelist): import os + import stat import errno import shutil + tmpfilesdir = '%s%s/tmpfiles.d' % (root, d.getVar('libdir')) + # Remove content that is no longer needed. for entry in stateless_rm: old = os.path.join(root, 'etc', entry) @@ -71,29 +187,114 @@ def stateless_mangle(d, root, docdir, stateless_mv, stateless_rm, dirwhitelist, # Move away files. Default target is docdir, but others can # be set by appending = to the entry, as in - # tmpfiles.d=libdir/tmpfiles.d + # tmpfiles.d=libdir/tmpfiles.d. "factory" as target adds + # the file to those restored by systemd if missing. for entry in stateless_mv: paths = entry.split('=', 1) etcentry = paths[0] old = os.path.join(root, 'etc', etcentry) if os.path.exists(old) or os.path.islink(old): + factory = False + tmpfiles_before = [] if len(paths) > 1: - new = root + paths[1] + if paths[1] == 'factory' or paths[1].startswith('factory:'): + new = root + '/usr/share/factory/etc/' + paths[0] + factory = True + parts = paths[1].split(':', 1) + if len(parts) > 1: + tmpfiles_before = parts[1].split(',') + (paths[1].split(':', 1)[1:] or [''])[0].split(',') + else: + new = root + paths[1] else: new = os.path.join(docdir, entry) destdir = os.path.dirname(new) bb.utils.mkdirhier(destdir) # Also handles moving of directories where the target already exists, by - # moving the content. When moving a relative symlink the target gets updated. + # moving the content. Symlinks are made relative to the target + # directory. + oldtop = old + moved = [] def move(old, new): bb.note('stateless: moving %s to %s' % (old, new)) - if os.path.isdir(new): + moved.append('/' + os.path.relpath(old, root)) + if os.path.islink(old): + link = os.readlink(old) + if link.startswith('/'): + target = root + link + else: + target = os.path.join(os.path.dirname(old), link) + target = os.path.normpath(target) + if not factory and os.path.relpath(target, oldtop).startswith('../'): + # Target outside of the root of what we are moving, + # so the target must remain the same despite moving + # the symlink itself. + link = os.path.relpath(target, os.path.dirname(new)) + else: + # Target also getting moved or the symlink will be restored + # at its current place, so keep link relative + # to where it is now. + link = os.path.relpath(target, os.path.dirname(old)) + if os.path.lexists(new): + os.unlink(new) + if not factory and (link == '/dev/null' or link.endswith('../dev/null')): + # Special case symlink to /dev/null (for example, /etc/tmpfiles.d/home.conf -> /dev/null): + # this is used to erase system defaults via local image settings. As we are now merging + # with the non-factory system defaults, we can simply erase the file and not + # create the symlink. + pass + else: + os.symlink(link, new) + os.unlink(old) + elif os.path.isdir(old): + if os.path.exists(new): + if not os.path.isdir(new): + bb.fatal('stateless: moving directory %s to non-directory %s not supported' % (old, new)) + else: + # TODO (?): also copy xattrs + os.mkdir(new) + shutil.copystat(old, new) + stat = os.stat(old) + os.chown(new, stat.st_uid, stat.st_gid) for entry in os.listdir(old): move(os.path.join(old, entry), os.path.join(new, entry)) os.rmdir(old) else: os.rename(old, new) move(old, new) + if factory: + # Add new tmpfiles.d entry for the top-level directory. + with open(os.path.join(tmpfilesdir, 'stateless.conf'), 'a+') as f: + if os.path.islink(new): + # Symlinks have to be created with a special tmpfiles.d entry. + link = os.readlink(new) + os.unlink(new) + f.write('L /etc/%s - - - - %s\n' % (etcentry, link)) + else: + f.write('C /etc/%s - - - - -\n' % etcentry) + # We might have moved an entry for which systemd (or something else) + # already had a tmpfiles.d entry. We need to remove that other entry + # to ensure that ours is used instead. + for file in os.listdir(tmpfilesdir): + if file.endswith('.conf') and file != 'stateless.conf': + with open(os.path.join(tmpfilesdir, file), 'r+') as f: + lines = [] + for line in f.readlines(): + parts = line.split() + if len(parts) >= 2 and parts[1] in moved: + line = '# replaced by stateless.conf entry: ' + line + lines.append(line) + f.seek(0) + f.write(''.join(lines)) + # Ensure that the listed service(s) start after tmpfiles.d setup. + if tmpfiles_before: + service_d_dir = '%s%s/systemd-tmpfiles-setup.service.d' % (root, d.getVar('systemd_system_unitdir')) + bb.utils.mkdirhier(service_d_dir) + conf_file = os.path.join(service_d_dir, 'stateless.conf') + with open(conf_file, 'a') as f: + if f.tell() == 0: + f.write('[Unit]\n') + f.write('Before=%s\n' % ' '.join(tmpfiles_before)) # Remove /etc if all that's left are directories. # Some directories are expected to exists (for example, @@ -102,18 +303,24 @@ def stateless_mangle(d, root, docdir, stateless_mv, stateless_rm, dirwhitelist, # removed. etcdir = os.path.join(root, 'etc') def tryrmdir(path): - if is_package and \ - path.endswith('/etc/modprobe.d') or \ - path.endswith('/etc/modules-load.d'): - # Expected to exist by kernel-module-split.bbclass - # which will clean it itself. - return - if stateless_is_whitelisted(path[len(etcdir) + 1:], dirwhitelist): + entry = path[len(etcdir) + 1:] + if stateless_is_whitelisted(entry, dirwhitelist): bb.note('stateless: keeping white-listed directory %s' % path) return - bb.note('stateless: removing dir %s' % path) + bb.note('stateless: removing dir %s (%s not in %s)' % (path, entry, dirwhitelist)) + path_stat = os.stat(path) try: os.rmdir(path) + # We may have moved some content into the tmpfiles.d factory, + # and that then depends on re-creating these directories. + etcentry = os.path.relpath(path, etcdir) + if etcentry != '.': + with open(os.path.join(tmpfilesdir, 'stateless.conf'), 'a') as f: + f.write('D /etc/%s 0%o %d %d - -\n' % + (etcentry, + stat.S_IMODE(path_stat.st_mode), + path_stat.st_uid, + path_stat.st_gid)) except OSError as ex: bb.note('stateless: removing dir failed: %s' % ex) if ex.errno != errno.ENOTEMPTY: @@ -129,154 +336,3 @@ def stateless_mangle(d, root, docdir, stateless_mv, stateless_rm, dirwhitelist, for file in files: bb.note('stateless: /etc not empty: %s' % os.path.join(root, file)) tryrmdir(etcdir) - - -# Modify ${D} after do_install and before do_package resp. do_populate_sysroot. -do_install[postfuncs] += "stateless_mangle_package" -python stateless_mangle_package() { - pn = d.getVar('PN', True) - if pn in (d.getVar('STATELESS_PN_WHITELIST', True) or '').split(): - return - installdir = d.getVar('D', True) - docdir = installdir + os.path.join(d.getVar('docdir', True), pn, 'etc') - whitelist = (d.getVar('STATELESS_ETC_DIR_WHITELIST', True) or '').split() - - stateless_mangle(d, installdir, docdir, - (d.getVar('STATELESS_MV', True) or '').split(), - (d.getVar('STATELESS_RM', True) or '').split(), - whitelist, - True) -} - -# Check that nothing is left in /etc. -PACKAGEFUNCS += "stateless_check" -python stateless_check() { - pn = d.getVar('PN', True) - if pn in (d.getVar('STATELESS_PN_WHITELIST', True) or '').split(): - return - whitelist = (d.getVar('STATELESS_ETC_WHITELIST', True) or '').split() - import os - sane = True - for pkg, files in pkgfiles.items(): - pkgdir = os.path.join(d.getVar('PKGDEST', True), pkg) - for file in files: - targetfile = file[len(pkgdir):] - if targetfile.startswith('/etc/') and \ - not stateless_is_whitelisted(targetfile[len('/etc/'):], whitelist): - bb.warn("stateless: %s should not contain %s" % (pkg, file)) - sane = False - if not sane: - d.setVar("QA_SANE", "") -} - -QAPATHTEST[stateless] = "stateless_qa_check_paths" -def stateless_qa_check_paths(file,name, d, elf, messages): - """ - Check for deprecated paths that should no longer be used. - """ - - if os.path.islink(file): - return - - # Ignore ipk and deb's CONTROL dir - if file.find(name + "/CONTROL/") != -1 or file.find(name + "/DEBIAN/") != -1: - return - - bad_paths = d.getVar('STATELESS_DEPRECATED_PATHS', True).split() - if bad_paths: - import subprocess - import pipes - cmd = "strings -a %s | grep -F '%s' | sort -u" % (pipes.quote(file), '\n'.join(bad_paths)) - s = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = s.communicate() - # Cannot check return code, some of them may get lost because we use a pipe - # and cannot rely on bash's pipefail. Instead just check for unexpected - # stderr content. - if stderr: - bb.fatal('Checking %s for paths deprecated via STATELESS_DEPRECATED_PATHS failed:\n%s' % (file, stderr)) - if stdout: - package_qa_add_message(messages, "stateless", "%s: %s contains paths deprecated in a stateless configuration: %s" % (name, package_qa_clean_path(file, d), stdout)) -do_package_qa[vardeps] += "stateless_qa_check_paths" - -python () { - # The bitbake cache must be told explicitly that changes in the - # directories have an effect on the recipe. Otherwise adding - # or removing patches or whole directories does not trigger - # re-parsing and re-building. - import os - patchdir = d.expand('${STATELESS_PATCHES_BASE}/${PN}') - bb.parse.mark_dependency(d, patchdir) - if os.path.isdir(patchdir): - patches = os.listdir(patchdir) - if patches: - filespath = d.getVar('FILESPATH', True) - d.setVar('FILESPATH', filespath + ':' + patchdir) - srcuri = d.getVar('SRC_URI', True) - d.setVar('SRC_URI', srcuri + ' ' + ' '.join(['file://' + x for x in sorted(patches)])) - - # Dynamically reconfigure the package to use /usr instead of /etc for - # configuration files. - relocate = d.getVar('STATELESS_RELOCATE', True) - if relocate != 'False': - defaultsdir = d.expand('${datadir}/defaults') if relocate == 'True' else relocate - d.setVar('sysconfdir', defaultsdir) - d.setVar('EXTRA_OECONF', d.getVar('EXTRA_OECONF', True) + " --sysconfdir=" + defaultsdir) -} - -# Several post-install scripts modify /etc. -# For example: -# /etc/shells - gets extended when installing a shell package -# /etc/passwd - adduser in postinst extends it -# /etc/systemd/system - has several .wants entries -# -# We fix this directly after the write_image_manifest command -# in the ROOTFS_POSTUNINSTALL_COMMAND. -# -# However, that is very late, so changes made by a ROOTFS_POSTPROCESS_COMMAND -# (like setting an empty root password) become part of the system, -# which might not be intended in all cases. -# -# It would be better to do this directly after installing with -# ROOTFS_POSTINSTALL_COMMAND += "stateless_mangle_rootfs;" -# However, opkg then becomes unhappy and causes failures in the -# *_manifest commands which get executed later: -# -# ERROR: Cannot get the installed packages list. Command '.../opkg -f .../refkit-image-minimal/1.0-r0/opkg.conf -o .../refkit-image-minimal/1.0-r0/rootfs --force_postinstall --prefer-arch-to-version status' returned 0 and stderr: -# Collected errors: -# * file_md5sum_alloc: Failed to open file .../refkit-image-minimal/1.0-r0/rootfs/etc/hosts: No such file or directory. -# -# ERROR: Function failed: write_package_manifest -# -# TODO: why does opkg complain? /etc/hosts is listed in CONFFILES of netbase, -# so it should be valid to remove it. If we can fix that and ensure that -# all /etc files are marked as CONFFILES (perhaps by adding that as -# default for all packages), then we can use ROOTFS_POSTINSTALL_COMMAND -# again. -ROOTFS_POSTUNINSTALL_COMMAND_append = "stateless_mangle_rootfs;" - -python stateless_mangle_rootfs () { - pn = d.getVar('PN', True) - if pn in (d.getVar('STATELESS_PN_WHITELIST', True) or '').split(): - return - - rootfsdir = d.getVar('IMAGE_ROOTFS', True) - docdir = rootfsdir + d.getVar('datadir', True) + '/doc/etc' - whitelist = (d.getVar('STATELESS_ETC_WHITELIST', True) or '').split() - stateless_mangle(d, rootfsdir, docdir, - (d.getVar('STATELESS_MV_ROOTFS', True) or '').split(), - (d.getVar('STATELESS_RM_ROOTFS', True) or '').split(), - whitelist, - False) - import os - etcdir = os.path.join(rootfsdir, 'etc') - valid = True - for dirpath, dirnames, filenames in os.walk(etcdir): - for entry in filenames + [x for x in dirnames if os.path.islink(x)]: - fullpath = os.path.join(dirpath, entry) - etcentry = fullpath[len(etcdir) + 1:] - if not stateless_is_whitelisted(etcentry, whitelist): - bb.warn('stateless: rootfs should not contain %s' % fullpath) - valid = False - if not valid: - bb.fatal('stateless: /etc not empty') -} diff --git a/meta-refkit-core/classes/systemd-sysusers.bbclass b/meta-refkit-core/classes/systemd-sysusers.bbclass index a6bdcbc5cc..215f9ee23b 100644 --- a/meta-refkit-core/classes/systemd-sysusers.bbclass +++ b/meta-refkit-core/classes/systemd-sysusers.bbclass @@ -70,6 +70,14 @@ systemd_sysusers_create () { ;; esac done + + # We are done with the file, it's not needed anymore. + # This is also a workaround for systemd creating a "nobody" + # group for the "u nobody" entry in basic.conf. + # The code above doesn't do that because there is a "nobody" + # user already in /etc/passwd. Probably the base configuration + # should have a similar group (https://bugzilla.yoctoproject.org/show_bug.cgi?id=11766). + rm "$conf" fi done } diff --git a/meta-refkit-core/conf/distro/include/refkit-config.inc b/meta-refkit-core/conf/distro/include/refkit-config.inc index 4040cfb747..1bf4e96a90 100644 --- a/meta-refkit-core/conf/distro/include/refkit-config.inc +++ b/meta-refkit-core/conf/distro/include/refkit-config.inc @@ -58,6 +58,15 @@ REFKIT_DEFAULT_DISTRO_FEATURES += "refkit-config" # Enable OSTree system update support. REFKIT_DEFAULT_DISTRO_FEATURES += "ostree" +# Reconfigure and/or patch some recipes to support "stateless" images +# better (stateless = configuration in /etc can be created locally or +# isn't needed at all). Note that the actual changes are defined by +# the stateless*.inc files included by a distro config like +# refkit.conf, i.e. merely including refkit-config.inc does not +# have much effect even when "stateless" is enabled as distro feature. +REFKIT_DEFAULT_DISTRO_FEATURES += "stateless" +require conf/distro/include/stateless.inc + # Remove currently unsupported distro features from global defaults REFKIT_DEFAULT_DISTRO_FEATURES_REMOVE += "x11 3g" diff --git a/meta-refkit-core/conf/distro/include/stateless-factory.inc b/meta-refkit-core/conf/distro/include/stateless-factory.inc new file mode 100644 index 0000000000..eba7eedf66 --- /dev/null +++ b/meta-refkit-core/conf/distro/include/stateless-factory.inc @@ -0,0 +1,155 @@ +# This include file is a proof-of-concept for using systemd's +# tmpfiles.d mechanism to populate /etc from factory defaults. +# +# This approach has several drawbacks: +# - When /etc gets populated this way, the resulting files +# look to a system update mechanism like OSTree and swupd +# like they were created locally by an administor, with +# the result that they won't be touched during a system +# update even when the factory defaults change. +# - It only works for files which are not needed by +# by systemd itself during the early boot phase, otherwise +# the device will be in a different state after the initial +# boot compared to the state after the first reboot. +# For example, /etc/hostname gets moved into factory +# defaults here, but then only becomes effective after +# a reboot. +# +# Overall it is better to remove the need for files in /etc entirely +# (difficult, often needs patches) or use a system update mechanism +# which has at least some support for /etc (like OSTree, or swupd +# when built in non-stateless mode). + +# thermald: upstart init file not needed, +# config file can be in factory. +# TODO: remove the file during packaging +STATELESS_MV_ROOTFS += " \ + init/thermald.conf=factory \ +" +STATELESS_MV_ROOTFS += " \ + thermald=factory \ +" + +# Moving /etc/hostname has the effect that a reboot is required +# before the configured hostname becomes effective again. As not +# much depends on it, that seems a reasonable default. +STATELESS_MV_ROOTFS += " \ + hostname=factory \ +" + +# By ensuring that udevd starts after tmpfiles, we can move +# its main config file into the factory defaults. +STATELESS_MV_ROOTFS += " \ + udev/udev.conf=factory:systemd-udevd.service \ +" + +# Move away ld.so.conf and let systemd's factory reset mechanism re-create +# it during boot. For this to work reliably, ldconfig.service must run +# after systemd-tmpfiles-setup.service. Normally they run in parallel. +STATELESS_MV_ROOTFS += " \ + ld.so.conf=factory:ldconfig.service \ +" + +# systemd/system.conf and systemd/journald.conf can be moved to +# /usr/share/doc if and only if they only contains the default, +# commented out values, because non-default values must already be set +# before these daemons start. +# +# For journald.conf that is problematic, because systemd_232.bb +# changes journald.conf instead of compiling systemd with different +# defaults. That could be changed. For now we ignore those +# modifications and thus accept that the first boot without +# journald.conf will not run quite as it would normally. +# +# Service settings can be moved to /usr because they are part +# of the system. +# +# All remaining systemd config files may or may not have been +# modified and thus get treated as factory defaults. +STATELESS_POSTPROCESS += " stateless_mv_systemd_conf;" +stateless_mv_systemd_conf () { + for config in system.conf journald.conf; do + if [ -e ${IMAGE_ROOTFS}${sysconfdir}/systemd/$config ]; then + if settings=`grep -v -e '^\[.*\]$' -e '^#' -e '^$' -e '^RuntimeMaxUse=' -e '^ForwardToSyslog=' ${IMAGE_ROOTFS}/etc/systemd/$config`; then + bbfatal "stateless: ${IMAGE_ROOTFS}/etc/systemd/$config contains more than just comments, cannot remove:\n$settings" + fi + mkdir -p ${IMAGE_ROOTFS}${datadir}/doc/etc/systemd + mv ${IMAGE_ROOTFS}${sysconfdir}/systemd/$config ${IMAGE_ROOTFS}${datadir}/doc/etc/systemd + fi + done +} +STATELESS_MV_ROOTFS += " \ + systemd/system=${systemd_system_unitdir} \ + xdg/systemd=factory \ + systemd=factory \ +" + +# Several files in /etc/ssl can become factory defaults. +# /etc/ssl/certs and /etc/ssl itself will be dealt with below. +STATELESS_MV_ROOTFS += " \ + ssl/openssl.cnf=factory \ + ssl/openssl.cnf.real=factory \ + ssl/private=factory \ +" + +# We could just dump /etc/ssl/certs entirely into the factory +# defaults, but that sounds redundant, because the content +# is already generated from read-only system content. Instead, +# we extend systemd-tmpfiles-setup.service so that it +# also runs update-ca-certificates. +STATELESS_POSTPROCESS += " stateless_rm_etc_ssl_certs;" +stateless_rm_etc_ssl_certs () { + if [ -e ${IMAGE_ROOTFS}${sbindir}/update-ca-certificates ] && + [ -e ${IMAGE_ROOTFS}${systemd_system_unitdir}/systemd-tmpfiles-setup.service ]; then + echo "ExecStartPost=/bin/sh -c '[ -e ${sysconfdir}/ssl/certs/ca-certificates.crt ] || ${sbindir}/update-ca-certificates'" >>${IMAGE_ROOTFS}${systemd_system_unitdir}/systemd-tmpfiles-setup.service + rm -rf ${IMAGE_ROOTFS}${sysconfdir}/ssl/certs + # If empty now, /etc/ssl can be removed, too. + if rmdir ${IMAGE_ROOTFS}${sysconfdir}/ssl; then + echo "d ${sysconfdir}/ssl 0755 root root - -" >>${IMAGE_ROOTFS}${libdir}/tmpfiles.d/stateless.conf + fi + echo "d ${sysconfdir}/ssl/certs 0755 root root - -" >>${IMAGE_ROOTFS}${libdir}/tmpfiles.d/stateless.conf + fi +} + +# OE chooses the actual resolver (for example, ConnMan vs systemd) at +# build time by symlinking /etc/resolv.conf to the actual .conf file. +# We need to preserve that choice. +STATELESS_MV_ROOTFS += " \ + resolv.conf=factory \ +" + +# Various things that systemd and journald do not need when they +# start. +STATELESS_MV_ROOTFS += " \ + asound.conf=factory \ + bluetooth=factory \ + busybox.links.nosuid=factory \ + busybox.links.suid=factory \ + ca-certificates.conf=factory \ + dbus-1=factory \ + default=factory \ + environment=factory \ + filesystems=factory \ + grub.d=factory \ + host.conf=factory \ + inputrc=factory \ + issue=factory \ + issue.net=factory \ + libnl=factory \ + mke2fs.conf=factory \ + motd=factory \ + network=factory \ + nftables=factory \ + os-release=factory \ + profile=factory \ + request-key.conf=factory \ + resolv-conf.connman=factory \ + resolv-conf.systemd=factory \ + security=factory \ + ssh=factory \ + skel=factory \ + timestamp=factory \ + udhcpc.d=factory \ + version=factory \ + wpa_supplicant.conf=factory \ +" diff --git a/meta-refkit-core/conf/distro/include/stateless-login.inc b/meta-refkit-core/conf/distro/include/stateless-login.inc new file mode 100644 index 0000000000..d53ea57a38 --- /dev/null +++ b/meta-refkit-core/conf/distro/include/stateless-login.inc @@ -0,0 +1,54 @@ +# Moves login-related files from /etc to /usr/share/doc/etc. Relies +# on sane defaults in the binaries which read these files. Don't use +# this include file if the distro actually depends on non-standard +# settings in those files! +# +# BEWARE: depends on non-upstream patch, therefore not currently +# used in IoT Refkit and might not even compile. + +STATELESS_MV_ROOTFS += " \ + login.defs \ + securetty \ + shells \ +" + +# Enable logins without /etc/login.defs. +STATELESS_SRC_append_pn-shadow = " \ + https://raw.githubusercontent.com/clearlinux-pkgs/shadow/2aae81d2f493e340f454e6888c79f71c0414726c/0001-Do-not-bail-out-on-missing-login.defs.patch \ + 7bf3f3df680fe1515deca2e7bc1715759616f101156650c95172366a79817662 \ + https://raw.githubusercontent.com/clearlinux-pkgs/shadow/2aae81d2f493e340f454e6888c79f71c0414726c/stateless-login.patch \ + 3bb9bc5936111fac2cfd9723423281c98533740c5ca564152488d9ba33021cc5 \ +" + +# Use /usr/share/pam.d instead of /usr/lib/pam.d (for the sake of consistency?) +# and move /etc files to it. We must prevent systemd from re-creating the files +# from its own builtin copies. +STATELESS_SRC_append_pn-libpam = " \ + https://raw.githubusercontent.com/clearlinux-pkgs/Linux-PAM/b71399c80514afa9411b00aef2be721338a77893/0001-libpam-Keep-existing-pamdir-for-transition.patch \ + 25761101f785878dc7817344f484f670de5723df2eccc17dad9236af446cb890 \ +" +STATELESS_MV_ROOTFS += "pam.d=${datadir}/pam.d" +STATELESS_POSTPROCESS += " stateless_rm_systemd_pamd_factory;" +stateless_rm_systemd_pamd_factory () { + rm -rf ${IMAGE_ROOTFS}${datadir}/factory/etc/pam.d + if [ -f ${IMAGE_ROOTFS}${libdir}/tmpfiles.d/etc.conf ]; then + sed -i -e 's;^\(C */etc/pam.d *.*\);# stateless: \1;' \ + ${IMAGE_ROOTFS}${libdir}/tmpfiles.d/etc.conf + fi +} + +# Allow logins without /etc/login.defs, /etc/securetty or /etc/shells. +STATELESS_SRC_append_pn-libpam = " \ + https://raw.githubusercontent.com/clearlinux-pkgs/Linux-PAM/0681d308b660919e6a7ee71be41397dbc8516519/0003-pam_env-Only-report-non-ENOENT-errors-for-env-file.patch \ + 5b6866931e70524ed29cc2b2f5abf31f732658441207d441ec00cbcb9f04833e \ + https://raw.githubusercontent.com/clearlinux-pkgs/Linux-PAM/aa0bf6295ec8faa96cad1094806a545aae03247e/0004-pam_shells-Support-a-stateless-configuration-by-defa.patch \ + 35d3ca298728aab229b1b82e01ae6b7d0f7be11b0e71c7d18d92ebc8069087aa \ +" + +# TODO (?): avoid log entry about "Couldn't open /etc/securetty" each time +# pam_securetty is used. Written for libpam 1.2.1, does not apply to 1.3.0 +# because the code was modified. Not particularly important as pam_securetty +# seems unused in OE-core. +#SRC_URI_append_pn-libpam = " \ +# https://raw.githubusercontent.com/clearlinux-pkgs/Linux-PAM/0681d308b660919e6a7ee71be41397dbc8516519/0001-pam_securetty-Do-not-report-non-fatal-documented-beh.patch \ +#" diff --git a/meta-refkit-core/conf/distro/include/stateless-nss-altfiles.inc b/meta-refkit-core/conf/distro/include/stateless-nss-altfiles.inc new file mode 100644 index 0000000000..5604a43c07 --- /dev/null +++ b/meta-refkit-core/conf/distro/include/stateless-nss-altfiles.inc @@ -0,0 +1,67 @@ +# Install nss-altfiles, activate it in nsswitch.conf, and move the +# STATELESS_ALTFILES (like passwd) from /etc into +# /usr/lib/defaults/etc. +# +# As we do this after other ROOTFS_POSTPROCESS_COMMAND, +# a default root password makes it into the read-only defaults +# which - if done - is probably intended for debug images. +# +# BEWARE: depends on non-upstream patches, therefore not currently +# used in IoT Refkit and might not even compile. + +STATELESS_EXTRA_INSTALL += "nss-altfiles" +STATELESS_POSTPROCESS += " stateless_activate_altfiles;" + +stateless_activate_altfiles () { + # This adds "altfiles" as fallback after "compat" or "files". + # Relies on nsswitch.conf, which may have been moved already + # by stateless_activate_nsswitch. + nsswitch_conf=${IMAGE_ROOTFS}/${sysconfdir}/nsswitch.conf + if ! [ -e $nsswitch_conf ]; then + nsswitch_conf=${IMAGE_ROOTFS}${datadir}/defaults/etc/nsswitch.conf + if ! [ -e $nsswitch_conf ]; then + bbfatal "nsswitch.conf neither in ${IMAGE_ROOTFS}/${sysconfdir} nor in ${IMAGE_ROOTFS}${datadir}/defaults/etc, require for nss-altfiles." + fi + fi + install -d ${IMAGE_ROOTFS}${datadir}/defaults/etc + sed -i -e 's/files/files altfiles/' -e 's/compat/compat altfiles/' $nsswitch_conf +} + +STATELESS_ALTFILES = "hosts services protocols rpc passwd group shadow gshadow" +STATELESS_MV_ROOTFS += " \ + ${@ ' '.join('%s=${datadir}/defaults/etc/%s' % (x,x) for x in '${STATELESS_ALTFILES}'.split())} \ +" + +# Do not bail out in "adduser" when /etc/passwd is missing. +STATELESS_SRC_append_pn-busybox = " \ + file://adduser-enable-use-without-etc-passwd.patch None \ +" + +# Teach shadow about altfiles in /usr/defaults/etc and /usr/defaults/skel. +# For example, setting a password will copy an existing entry from there into /etc. +STATELESS_SRC_append_pn-shadow = " \ + https://raw.githubusercontent.com/clearlinux-pkgs/shadow/2aae81d2f493e340f454e6888c79f71c0414726c/0003-Do-not-fail-on-missing-files-in-etc-create-them-inst.patch \ + 3df4182a48a60dc796a2472812adc1a96146c461e6951646c4baaf47e80ed943 \ + https://raw.githubusercontent.com/clearlinux-pkgs/shadow/2aae81d2f493e340f454e6888c79f71c0414726c/0004-Force-use-shadow-even-if-missing.patch \ + 8e744ae7779b64d7d9668dc2e9bbf42840dd4ed668b66c6bc22bd88837914bd5 \ + https://raw.githubusercontent.com/clearlinux-pkgs/shadow/2aae81d2f493e340f454e6888c79f71c0414726c/0005-Create-dbs-with-correct-permissions.patch \ + cb669ad9e99fba3672733524d4e8671b69a86d303f02d915580fc8af586c2aef \ + https://raw.githubusercontent.com/clearlinux-pkgs/shadow/2aae81d2f493e340f454e6888c79f71c0414726c/0006-Make-usermod-read-altfiles.patch \ + 618e1c6b80f03143c614c9338284cae7928b8fed0a726eed6d8b6f38fdb3d5e5 \ + https://raw.githubusercontent.com/clearlinux-pkgs/shadow/2aae81d2f493e340f454e6888c79f71c0414726c/stateless-adduser.patch \ + 8fff0b1c52712050b3652d26c8a5faf2acc4cf458964c04a6ca1d28d1d928f2e \ + https://raw.githubusercontent.com/clearlinux-pkgs/shadow/6d0c85ab07e6c7dd399953f3b9fc24947f910bc8/stateless-gpasswd.patch \ + e79a3fac817240ebe3144bab67e7ab5f1247b28b59310a13aa9f2cca33d20451 \ + https://raw.githubusercontent.com/clearlinux-pkgs/shadow/2aae81d2f493e340f454e6888c79f71c0414726c/stateless-useradd.patch \ + 6f47bd7c5df44a1c4dab1bd102c5a8f0f60cf40fd5c6b4c1afd6f7758f280162 \ + https://raw.githubusercontent.com/clearlinux-pkgs/shadow/d34359528e24569457b8ee8f66d6f2991a291c67/stateless-usermod.patch \ + af825f9c02834eb7ec34f3ef4c1db0dbc2aed985d02e1c3bc6e8deba5f4ebf68 \ +" + +# Required for setting root password when /etc is empty, because +# otherwise PAM's "is changing the password allowed" check fails, +# leading to a "permission denied" error before the password prompt. +STATELESS_SRC_append_pn-libpam = " \ + https://raw.githubusercontent.com/clearlinux-pkgs/Linux-PAM/b71399c80514afa9411b00aef2be721338a77893/0002-Support-altfiles-locations.patch \ + 53636e3e68a60cef4012735d881cffbd3e653b104e55d94d05826c48b8ec9830 \ +" diff --git a/meta-refkit-core/conf/distro/include/stateless-nsswitch.inc b/meta-refkit-core/conf/distro/include/stateless-nsswitch.inc new file mode 100644 index 0000000000..2864488b75 --- /dev/null +++ b/meta-refkit-core/conf/distro/include/stateless-nsswitch.inc @@ -0,0 +1,27 @@ +# Enables the use of /etc/nsswitch.conf as local override for +# the defaults in /usr/defaults/etc/nsswitch.conf. +# +# BEWARE: depends on non-upstream patch, therefore not currently +# used in IoT Refkit and might not even compile. + +# Beware that creating an /etc/nsswitch.conf without actual entries +# causes a segfault. Reported upstream: +# https://lists.clearlinux.org/pipermail/dev/2017-July/000927.html +STATELESS_SRC_append_pn-glibc = " \ + https://raw.githubusercontent.com/clearlinux-pkgs/glibc/e54b638ef6b5f838e99f1f055474ef2603dfce19/nsswitch-altfiles.patch \ + 82b66bc66d935aed845ae51d0ea7188dbc964ae17bda715f7114805ef5cc915d \ +" +stateless_activate_nsswitch () { + # nsswitch.conf gets moved to /usr and is not needed anymore + # in /etc (see stateless_glibc_altfiles_patch). + install -d ${IMAGE_ROOTFS}${datadir}/defaults/etc + mv ${IMAGE_ROOTFS}/${sysconfdir}/nsswitch.conf ${IMAGE_ROOTFS}${datadir}/defaults/etc/nsswitch.conf + + # We must not let systemd re-create it during boot with the systemd defaults. + if [ -f ${IMAGE_ROOTFS}${libdir}/tmpfiles.d/etc.conf ]; then + sed -i -e 's;^\(C */etc/nsswitch.conf *.*\);# stateless: \1;' \ + ${IMAGE_ROOTFS}${libdir}/tmpfiles.d/etc.conf + fi + rm -f ${IMAGE_ROOTFS}${datadir}/factory/etc/nsswitch.conf +} +STATELESS_POSTPROCESS += " stateless_activate_nsswitch;" diff --git a/meta-refkit-core/conf/distro/include/stateless-usr.inc b/meta-refkit-core/conf/distro/include/stateless-usr.inc new file mode 100644 index 0000000000..b05e6c0445 --- /dev/null +++ b/meta-refkit-core/conf/distro/include/stateless-usr.inc @@ -0,0 +1,52 @@ +# This include file moves files from /etc into /usr or removes them +# where the upstream components already support such a change. Except +# for a few minor differences noted below, there should be no change +# in semantic. + +# Disable creation of /etc/ld.so.cache in stateless images. The file +# gets already recreated by systemd anyway when booting. Has to be +# done by unsetting LDCONFIGDEPEND (checked by rootfs.py, which +# creates the ld.so.cache) for all IoT Reference OS Kit images, but not the +# refkit-initramfs, so we cannot set it unconditionally. +python () { + if bb.utils.contains('IMAGE_FEATURES', 'stateless', True, False, d): + d.setVar('LDCONFIGDEPEND', '') +} + +# We can use the pre-generated hwdb.bin as OS default while still +# allowing the creation of an updated version in /etc later on. +# systemd-update-done.service will only run when there is +# something to update in /etc and there are rules in /etc, so +# we clean that up, too. +STATELESS_MV_ROOTFS += " \ + udev/hwdb.bin=${base_libdir}/udev/hwdb.bin \ + udev/hwdb.d=${base_libdir}/udev/hwdb.d \ +" + +# Anything related to tmpfiles.d in /etc can be considered part of the +# OS and thus be moved to /usr/lib. This includes /etc files which are +# named exactly like existing files under /usr/lib: the ones from +# /usr/lib get overwritten, which preserves the semantic that /etc has +# higher priority. +STATELESS_MV_ROOTFS += " \ + tmpfiles.d=${libdir}/tmpfiles.d \ +" + +# Similar for udev. There's just a slight change of semantic: +# entries in /etc override those from /run, which they no longer +# do after being moved to /usr/lib - shouldn't matter in practice. +STATELESS_MV_ROOTFS += " \ + udev/rules.d=${base_libdir}/udev/rules.d \ +" + +# Move /etc/terminfo to /lib/terminfo. That's still going to be +# used before /usr/share/terminfo. +STATELESS_MV_ROOTFS += " \ + terminfo=${base_libdir}/terminfo \ +" + +# systemd-modules-load.service supports /etc/modules-load.d as well +# as /usr/lib/modules-load.d. +STATELESS_MV_ROOTFS += " \ + modules-load.d=${libdir}/modules-load.d \ +" diff --git a/meta-refkit-core/conf/distro/include/stateless.inc b/meta-refkit-core/conf/distro/include/stateless.inc index ef3b45e40d..86983da127 100644 --- a/meta-refkit-core/conf/distro/include/stateless.inc +++ b/meta-refkit-core/conf/distro/include/stateless.inc @@ -1,67 +1,59 @@ -INHERIT += "stateless" - -########################################################################### - -# Temporary overrides until IoT Reference OS Kit is fully stateless. - -# This entry here allows everything in /etc because IoT Reference OS Kit is -# not actually stateless yet. We merely use the stateless.bbclass -# to remove files from /etc which get written on the device -# and thus must be excluded from images and, more importantly, -# swupd bundles. -STATELESS_ETC_WHITELIST += "*" +# This file is meant to be included in a distro configuration. +# It adds support for the "stateless" distro and image feature, +# without actually turning them on. +# +# Addditional .inc files define the actual changes that a +# distro might want to apply. -# Empty directories must be kept. -STATELESS_ETC_DIR_WHITELIST += "*" +# The stateless.bbclass modifies how some recipes are built +# when "stateless" is in DISTRO_FEATURES, so it needs to +# be inherited globally. This can be removed once STATELESS_SRC +# is no longer needed. +INHERIT += "stateless" -########################################################################### +# Required to find local patches. +FILESEXTRAPATHS_prepend = "${@ bb.utils.contains('DISTRO_FEATURES', 'stateless', '${META_REFKIT_CORE_BASE}/files/stateless/${PN}:', '', d) }" -# As step towards full stateless IoT Reference OS Kit, we now -# treat some files in /etc as conceptually read-only (i.e. neither -# modified by the OS at runtime nor by an admin). Anything contained -# in the rootfs directories will get bundled and added or updated when -# running "swupd update". -# -# The implication is that we must keep certain files out of the rootfs -# which do get modified at runtime, because otherwise there are -# "swupd verify" failures. +# Some entries in /etc simply have to be there, so we whitelist them +# to avoid failing the QA check for stateless images. -# mtab needs to be a symlink to /proc/mounts, probably forever. -# There is no point in patching that out of binaries, nor is there -# a need to customize it, so the symlink can remain there as read-only -# system component. +# mtab is a symlink to /proc/mounts. STATELESS_ETC_WHITELIST += "mtab" -# OE-core puts some files into /etc which systemd then later overwrites -# unconditionally via /usr/lib/tmpfile.d/etc.conf or creates dynamically -# (machine-id). Therefore we can remove the redundant files from our rootfs -# by not packaging them in the first place. -STATELESS_RM_pn-systemd += " \ - resolv.conf \ -" - -# machine-id has to be present in images at least as an empty file -# because we might boot with the rootfs read/only. Otherwise -# creating it during early boot fails (see -# systemd/src/core/machine-id-setup.c). +# /etc/machine-id could be removed if (and only if) the rootfs gets mounted rw. +# Note that this also triggers the ConditionFirstBoot, something +# that is not normally done in OE-core. It causes systemd to +# auto-enable units according to their [Install] sections, and +# at least for wpa_supplicant that is broken (https://www.reddit.com/r/archlinux/comments/4mnkyu/timeout_during_boot/?st=j03jwv0d&sh=14c1a955 +# http://lists.infradead.org/pipermail/hostap/2017-March/037330.html) +# Therefore we always keep it. STATELESS_ETC_WHITELIST += "machine-id" -# These files must be ignored by swupd. -STATEFUL_FILES += "/etc/machine-id" -SWUPD_FILE_BLACKLIST_append = " ${STATEFUL_FILES}" - -# Depend on the installed components and thus has to be computed on -# the device. Handled by systemd during booting or updates. -STATELESS_RM_ROOTFS += " \ - udev/hwdb.bin \ -" - -# Disable creation of /etc/ld.so.cache in images and bundles. The file -# gets already recreated by systemd anyway when booting. Has to be -# done by unsetting LDCONFIGDEPEND (checked by rootfs.py, which -# creates the ld.so.cache) for all IoT Reference OS Kit images, but not the -# refkit-initramfs, so we cannot set it unconditionally. +# /etc/fstab can be removed only under special circumstances: +# - no local file systems besides root +# - rootfs gets mounted rw immediately +# - no additional special mount options for root that need +# to be applied via remount +# +# This is too complicated to check here, therefore /etc/fstab +# is left in place by default. A distro where fstab is known +# to be not needed can do: +# STATELESS_RM_ROOTFS += "fstab" +# STATELESS_ETC_WHITELIST_remove = "fstab" +STATELESS_ETC_WHITELIST += "fstab" + +# If we want to be stateless, override the /etc/build default. +# It currently gets created after STATELESS_MV_ROOTFS, so +# we can't do it that way. python () { - if bb.data.inherits_class('refkit-image', d): - d.setVar('LDCONFIGDEPEND', '') + buildinfo_file = d.getVar('IMAGE_BUILDINFO_FILE') + if bb.utils.contains('IMAGE_FEATURES', 'stateless', True, False, d) and \ + buildinfo_file is not None and buildinfo_file == '/etc/build': + d.setVar('IMAGE_BUILDINFO_FILE', '${libdir}/build') } + +# /etc/xdg/systemd/user symlinks to this, so keep the directory even when empty. +STATELESS_ETC_DIR_WHITELIST += "systemd/user" + +# Likewise for /usr/lib/ssl/certs -> /etc/ssl/certs and ssl/private. +STATELESS_ETC_DIR_WHITELIST += "ssl/certs ssl/private" diff --git a/meta-refkit-core/files/stateless/busybox/adduser-enable-use-without-etc-passwd.patch b/meta-refkit-core/files/stateless/busybox/adduser-enable-use-without-etc-passwd.patch new file mode 100644 index 0000000000..57ffec7c71 --- /dev/null +++ b/meta-refkit-core/files/stateless/busybox/adduser-enable-use-without-etc-passwd.patch @@ -0,0 +1,55 @@ +From 10a3e186e7dd5b8e470759346947bd7d99fba0e0 Mon Sep 17 00:00:00 2001 +From: Patrick Ohly +Date: Mon, 6 Mar 2017 16:24:16 +0100 +Subject: [PATCH 1/1] adduser: enable use without /etc/passwd + +The utility code which rewrites /etc/passwd or /etc/shadow assumes +that the files already exist. That is not the case in a stateless +system where nss-altfiles is used to read system users from /usr/share. + +Now the code falls back to creating the files instead of failing. + +Upstream-Status: Pending + +Signed-off-by: Patrick Ohly +--- + libbb/update_passwd.c | 8 +++++--- + 1 file changed, 5 insertions(+), 3 deletions(-) + +diff --git a/libbb/update_passwd.c b/libbb/update_passwd.c +index a2004f4..7553920 100644 +--- a/libbb/update_passwd.c ++++ b/libbb/update_passwd.c +@@ -88,6 +88,7 @@ int FAST_FUNC update_passwd(const char *filename, + int new_fd; + int i; + int changed_lines; ++ const char *real_filename; + int ret = -1; /* failure */ + /* used as a bool: "are we modifying /etc/shadow?" */ + #if ENABLE_FEATURE_SHADOWPASSWDS +@@ -96,7 +97,8 @@ int FAST_FUNC update_passwd(const char *filename, + # define shadow NULL + #endif + +- filename = xmalloc_follow_symlinks(filename); ++ real_filename = xmalloc_follow_symlinks(filename); ++ filename = real_filename ? real_filename : strdup(filename); + if (filename == NULL) + return ret; + +@@ -109,9 +111,9 @@ int FAST_FUNC update_passwd(const char *filename, + name_colon = xasprintf("%s:", name ? name : ""); + + if (shadow) +- old_fp = fopen(filename, "r+"); ++ old_fp = fopen(filename, "a+"); + else +- old_fp = fopen_or_warn(filename, "r+"); ++ old_fp = fopen_or_warn(filename, "a+"); + if (!old_fp) { + if (shadow) + ret = 0; /* missing shadow is not an error */ +-- +2.11.0 + diff --git a/meta-refkit-core/lib/oeqa/selftest/cases/refkit_ostree.py b/meta-refkit-core/lib/oeqa/selftest/cases/refkit_ostree.py index 705bc8a27d..26048dad3c 100644 --- a/meta-refkit-core/lib/oeqa/selftest/cases/refkit_ostree.py +++ b/meta-refkit-core/lib/oeqa/selftest/cases/refkit_ostree.py @@ -26,6 +26,13 @@ class RefkitOSTreeUpdateBase(SystemUpdateBase): # slirp network. OSTREE_SERVER = '10.0.2.100:8080' + def __init__(self, *args, **kwargs): + # Although we have the "stateless" distro feature, nss-altfiles and the associated + # patches are not active and thus creating users locally prevents further updates + # of /etc/passwd as part of a system update. + self.IMAGE_MODIFY.LOCAL_USERS = False + super().__init__(*args, **kwargs) + def track_for_cleanup(self, name): """ Run a single test with NO_CLEANUP= oe-selftest to not clean up after the test. @@ -118,6 +125,11 @@ def test_update_all(self): """ self.do_update('test_update_all', self.IMAGE_MODIFY.UPDATES) + # TODO: a test that "ostree admin config-diff" doesn't show any unexpected modifications + # directly after booting. That's necessary because each such modification will prevent + # updating the modified file as part of a system update. One example for such a change + # was adding "nobody" to /etc/group (https://bugzilla.yoctoproject.org/show_bug.cgi?id=11766). + class RefkitOSTreeUpdateMeta(type): """ Generates individual instances of test_update_, one for each type of change. diff --git a/meta-refkit-core/lib/oeqa/selftest/systemupdate/systemupdatebase.py b/meta-refkit-core/lib/oeqa/selftest/systemupdate/systemupdatebase.py index 45fa638f68..a2bbae3b49 100644 --- a/meta-refkit-core/lib/oeqa/selftest/systemupdate/systemupdatebase.py +++ b/meta-refkit-core/lib/oeqa/selftest/systemupdate/systemupdatebase.py @@ -9,6 +9,7 @@ import base64 import pathlib import pickle +import subprocess class SystemUpdateModify(object): """ @@ -26,6 +27,7 @@ class SystemUpdateModify(object): 'files', 'home', 'kernel', + 'user', 'var', ] @@ -35,11 +37,14 @@ class SystemUpdateModify(object): # - modifying file locally before update which then must # be used instead of the updated system file ETC_FILES = [ - ( 'nsswitch.conf', 'edit' ), + ( 'host.conf', 'edit' ), ( 'ssl/openssl.cnf', 'symlink' ), ( 'ssh/sshd_config', None ), ] + # Users can be created locally without conflicting with system updates. + LOCAL_USERS = True + def modify_image_build(self, testname, updates, is_update): """ Returns additional settings that get stored in a .bbappend @@ -106,6 +111,7 @@ def modify_etc(self, testname, is_update, rootfs): if is_update: for file, operation in self.ETC_FILES: path = os.path.join(rootfs, 'etc', file) + assert os.path.exists(path) with open(path, 'ab') as f: f.write(b'\n# system update test\n') if operation == 'symlink': @@ -116,7 +122,7 @@ def verify_etc(self, testname, is_update, qemu, test): if not is_update: for file, operation in self.ETC_FILES: if operation == 'edit': - cmd = "echo '# edited locally' >>/etc/%s" % file + cmd = "ls -l /etc/{0} && echo '# edited locally' >>/etc/{0}".format(file) status, output = qemu.run_serial(cmd) test.assertEqual(1, status, 'Failed to run command "%s":\n%s' % (cmd, output)) else: @@ -155,6 +161,48 @@ def verify_var(self, testname, is_update, qemu, test): test.assertEqual(1, status, 'Failed to run command "%s":\n%s' % (cmd, output)) test.assertEqual(output, 'original: hello world') + def modify_user(self, testname, is_update, rootfs): + """ + Add a new system user during the update, and another real user on the device. + Both users are expected to be present after the update. + """ + if is_update: + subprocess.check_output('useradd --root %s --system test_update_sys_user' % rootfs, + shell=True, stderr=subprocess.STDOUT) + + def verify_user(self, testname, is_update, qemu, test): + if self.LOCAL_USERS: + if not is_update: + # Create a local user. This implies modifying /etc/passwd|shadow|group, + # which then must be handled by the update mechanism. + cmd = 'adduser -D test_update_local_user' + status, output = qemu.run_serial(cmd, timeout=30) + test.assertEqual(1, status, 'Failed to run command "%s":\n%s' % (cmd, output)) + + # Local user must exist before and after update, including the home directory. + cmd = 'su test_update_local_user -c "id -nu"' + status, output = qemu.run_serial(cmd) + test.assertEqual(1, status, 'Failed to run command "%s":\n%s' % (cmd, output)) + test.assertEqual(output, 'test_update_local_user') + cmd = 'su test_update_local_user -c "id -ng"' + status, output = qemu.run_serial(cmd) + test.assertEqual(1, status, 'Failed to run command "%s":\n%s' % (cmd, output)) + test.assertEqual(output, 'test_update_local_user') + cmd = 'diff -r /etc/skel /home/test_update_local_user' + status, output = qemu.run_serial(cmd) + test.assertEqual(1, status, 'Failed to run command "%s":\n%s' % (cmd, output)) + + if is_update: + # Check for new user created by update. + cmd = 'su test_update_sys_user -c "id -nu"' + status, output = qemu.run_serial(cmd) + test.assertEqual(1, status, 'Failed to run command "%s":\n%s' % (cmd, output)) + test.assertEqual(output, 'test_update_sys_user') + cmd = 'su test_update_sys_user -c "id -ng"' + status, output = qemu.run_serial(cmd) + test.assertEqual(1, status, 'Failed to run command "%s":\n%s' % (cmd, output)) + test.assertEqual(output, 'test_update_sys_user') + def _do_modifications(self, d, testname, updates, is_update): """ This code will run as part of a ROOTFS_POSTPROCESS_COMMAND. diff --git a/meta-refkit-core/recipes-core/nss-altfiles/nss-altfiles_git.bb b/meta-refkit-core/recipes-core/nss-altfiles/nss-altfiles_git.bb new file mode 100644 index 0000000000..f447087953 --- /dev/null +++ b/meta-refkit-core/recipes-core/nss-altfiles/nss-altfiles_git.bb @@ -0,0 +1,34 @@ +SUMMARY = "NSS module which can read user information from files in the same format as /etc/passwd and /etc/group stored in an alternate location" +LICENSE = "LGPL2.1" +LIC_FILES_CHKSUM = "file://COPYING;md5=fb1949d8d807e528c1673da700aff41f" + +SRC_URI = "git://github.com/aperezdc/nss-altfiles.git;protocol=https" + +# Modify these as desired +PV = "2.23.0+git${SRCPV}" +SRCREV = "42bec47544ad80d3e39342b11ea33da05ff9133d" + +S = "${WORKDIR}/git" + +SECURITY_CFLAGS = "${SECURITY_NO_PIE_CFLAGS}" + +# nss-altfiles build rules are defined in a custom Makefile. +# Additional compile flags can be set with a configure shell script. +# Compilation then must use normal make instead of oe_runmake, because +# the later causes (among others) CFLAGS and CPPFLAGS to be +# overridden, which would disable important parts of the build +# rules. +do_configure () { + ${S}/configure --datadir=${datadir}/defaults/etc --libdir=${libdir} --with-types=rpc,proto,hosts,network,service,pwd,grp,spwd,sgrp 'CFLAGS=${CFLAGS}' 'CXXFLAGS=${CXXFLAGS}' + # Reconfiguring with different options does not cause a rebuild. Must clean + # explicitly to achieve that. + make MAKEFLAGS= clean +} + +do_compile () { + make MAKEFLAGS= +} + +do_install () { + make MAKEFLAGS= install 'DESTDIR=${D}' +} diff --git a/meta-refkit/conf/distro/include/refkit-supported-recipes.txt b/meta-refkit/conf/distro/include/refkit-supported-recipes.txt index b612d3fc96..41a927ee03 100644 --- a/meta-refkit/conf/distro/include/refkit-supported-recipes.txt +++ b/meta-refkit/conf/distro/include/refkit-supported-recipes.txt @@ -333,6 +333,7 @@ object-recognition-msgs@ros-layer ocl-icd@refkit-computervision octomap-msgs@ros-layer octomap@ros-layer +nss-altfiles@refkit-core oe-swupd-helpers@meta-swupd opencl-headers@refkit-computervision opencv@openembedded-layer diff --git a/meta-refkit/conf/distro/refkit.conf b/meta-refkit/conf/distro/refkit.conf index ebb418749d..8b22c95c55 100644 --- a/meta-refkit/conf/distro/refkit.conf +++ b/meta-refkit/conf/distro/refkit.conf @@ -74,7 +74,30 @@ DISTRO_EXTRA_RRECOMMENDS += " ${REFKIT_DEFAULT_EXTRA_RRECOMMENDS}" # Distro settings potentially shared with other distros. require conf/distro/include/no-static-libs.inc require conf/distro/include/refkit_security_flags.inc -require conf/distro/include/stateless.inc + +# Turns build settings in /etc into image settings under /usr, +# without non-upstream patches. +require conf/distro/include/stateless-usr.inc + +# NOT used because it depends on non-upstream patches. +# Without this, creating local users conflicts with updating system +# users as part of a system update. Would be very nice to have. +# require conf/distro/include/stateless-nss-altfiles.inc + +# NOT used because it depends on non-upstream patches. +# Enables login without some files in /etc. Not so important. +# require conf/distro/include/stateless-login.inc + +# NOT used because it depends on non-upstream patches. +# Makes it possible to modify nsswitch.conf without +# conflicting with system settings. Not particularly +# important. +# require conf/distro/include/stateless-nsswitch.inc + +# Not used because it renders the /etc handling in OSTree +# and swupd useless: once /etc is populated, it remains +# unchanged even when system defaults change. +# require conf/distro/include/stateless-factory.inc # Include *and* enabled refkit configuration. Including # just refkit-config.inc would not enable the configuration