Enable the use of the OverlayFS for the LiveOS root filesystem.

Integrate the option to use an OverlayFS as the root filesystem
into the 90dmsquash-live module for testing purposes.

The rd.live.overlay.overlayfs option allows one to request an
OverlayFS overlay.  If a persistent overlay is detected at the
standard LiveOS path, the overlay & type detected will be used.

Tested primarily with transient, in-RAM overlay boots on vfat-
formatted Live USB devices, with persistent overlay directories
on ext4-formatted Live USB devices, and with embedded, persistent
overlay directories on vfat-formatted devices. (Persistent overlay
directories on a vfat-formatted device must be in an embedded
filesystem that supports the creation of trusted.* extended
attributes, and must provide valid d_type in readdir responses.)

The rd.live.overlay.readonly option, which allows a persistent
overlayfs to be mounted read only through a higher level transient
overlay directory, has been implemented through the multiple lower
layers feature of OverlayFS.

The default transient DM overlay size has been adjusted up to 32 GiB.
This change supports comparison of transient Device-mapper vs.
transient OverlayFS overlay performance.  A transient DM overlay
is a sparse file in memory, so this setting does not consume more
RAM for legacy applications.  It does permit a user to use all of
the available root filesystem storage, and fails gently when it is
consumed, as the available free root filesystem storage on a typical
LiveOS build is only a few GiB.  Thus, when booted on other-
than-small RAM systems, the transient DM overlay should not overflow.

OverlayFS offers the potential to use all of the available free RAM
or all of the available free disc storage (on non-vfat-devices)
in its overlay, even beyond the root filesystem available space,
because the OverlayFS root filesystem is a union of directories on
two different partitions.

This patch also cleans up some message spew at shutdown, shortens
the execution path in a couple of places, and uses persistent
DM targets where required.

modules.d/90dm/dm-shutdown.sh

@@ -14,7 +14,8 @@ _do_dm_shutdown() {
     return $ret
 }
 
-if command -v dmsetup >/dev/null; then
+if command -v dmsetup >/dev/null &&
+    [ "x$(dmsetup status)" != "xNo devices found" ]; then
     _do_dm_shutdown $1
 else
     :

modules.d/90dmsquash-live/apply-live-updates.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-if [ -b /dev/mapper/live-rw ] && [ -d /run/initramfs/live/updates -o -d /updates ]; then
+if [ -L /run/rootfsbase ] && [ -d /run/initramfs/live/updates -o -d /updates ]; then
     info "Applying updates to live image..."
     mount -o bind /run $NEWROOT/run
     # avoid overwriting symlinks (e.g. /lib -> /usr/lib) with directories

modules.d/90dmsquash-live/dmsquash-generator.sh

@@ -52,21 +52,30 @@ esac
 
 GENERATOR_DIR="$2"
 [ -z "$GENERATOR_DIR" ] && exit 1
-
 [ -d "$GENERATOR_DIR" ] || mkdir "$GENERATOR_DIR"
 
+getargbool 0 rd.live.overlay.overlayfs && overlayfs="yes"
+[ -e /xor_overlayfs ] && xor_overlayfs="yes"
 ROOTFLAGS="$(getarg rootflags)"
 {
     echo "[Unit]"
     echo "Before=initrd-root-fs.target"
     echo "[Mount]"
     echo "Where=/sysroot"
-    echo "What=/dev/mapper/live-rw"
-    [ -n "$ROOTFLAGS" ] && echo "Options=${ROOTFLAGS}"
+    if [ "$overlayfs$xor_overlayfs" = "yes" ]; then
+        echo "What=LiveOS_rootfs"
+        echo "Options=${ROOTFLAGS},lowerdir=/run/rootfsbase,upperdir=/run/overlayfs,workdir=/run/ovlwork"
+        echo "Type=overlay"
+        _dev=LiveOS_rootfs
+    else
+        echo "What=/dev/mapper/live-rw"
+        [ -n "$ROOTFLAGS" ] && echo "Options=${ROOTFLAGS}"
+        _dev=dev-mapper-live\x2drw
+    fi
 } > "$GENERATOR_DIR"/sysroot.mount
 
-mkdir -p "$GENERATOR_DIR/dev-mapper-live\x2drw.device.d"
+mkdir -p "$GENERATOR_DIR/$_dev.device.d"
 {
     echo "[Unit]"
     echo "JobTimeoutSec=3000"
-} > "$GENERATOR_DIR/dev-mapper-live\x2drw.device.d/timeout.conf"
+} > "$GENERATOR_DIR/$_dev.device.d/timeout.conf"

modules.d/90dmsquash-live/dmsquash-live-root.sh

@@ -28,9 +28,10 @@ getargbool 0 rd.live.overlay.readonly -d -y readonly_overlay && readonly_overlay
 overlay=$(getarg rd.live.overlay -d overlay)
 getargbool 0 rd.writable.fsimg -d -y writable_fsimg && writable_fsimg="yes"
 overlay_size=$(getarg rd.live.overlay.size=)
-[ -z "$overlay_size" ] && overlay_size=512
+[ -z "$overlay_size" ] && overlay_size=32768
 
 getargbool 0 rd.live.overlay.thin && thin_snapshot="yes"
+getargbool 0 rd.live.overlay.overlayfs && overlayfs="yes"
 
 # CD/DVD media check
 [ -b $livedev ] && fs=$(blkid -s TYPE -o value $livedev)
@@ -110,48 +111,94 @@ do_live_overlay() {
     setup=""
     if [ -n "$devspec" -a -n "$pathspec" -a -n "$overlay" ]; then
         mkdir -m 0755 /run/initramfs/overlayfs
+        opt=''
+        [ -n "$readonly_overlay" ] && opt=-r
         mount -n -t auto $devspec /run/initramfs/overlayfs || :
         if [ -f /run/initramfs/overlayfs$pathspec -a -w /run/initramfs/overlayfs$pathspec ]; then
-            losetup $OVERLAY_LOOPDEV /run/initramfs/overlayfs$pathspec
-            if [ -n "$reset_overlay" ]; then
-                dd if=/dev/zero of=$OVERLAY_LOOPDEV bs=64k count=1 conv=fsync 2>/dev/null
+            losetup $opt $OVERLAY_LOOPDEV /run/initramfs/overlayfs$pathspec
+            umount -l /run/initramfs/overlayfs || :
+            oltype=$(det_img_fs $OVERLAY_LOOPDEV)
+            if [ -z "$oltype" ] || [ "$oltype" = DM_snapshot_cow ]; then
+                if [ -n "$reset_overlay" ]; then
+                    info "Resetting the Device-mapper overlay."
+                    dd if=/dev/zero of=$OVERLAY_LOOPDEV bs=64k count=1 conv=fsync 2>/dev/null
+                fi
+                if [ -n "$overlayfs" ]; then
+                    unset -v overlayfs
+                    [ -n "$DRACUT_SYSTEMD" ] && reloadsysrootmountunit="yes"
+                fi
+                setup="yes"
+            else
+                mount -n -t $oltype $opt $OVERLAY_LOOPDEV /run/initramfs/overlayfs
+                if [ -d /run/initramfs/overlayfs/overlayfs ] &&
+                    [ -d /run/initramfs/overlayfs/ovlwork ]; then
+                    ln -s /run/initramfs/overlayfs/overlayfs /run/overlayfs$opt
+                    ln -s /run/initramfs/overlayfs/ovlwork /run/ovlwork$opt
+                    if [ -z "$overlayfs" ]; then
+                        overlayfs="yes"
+                        [ -n "$DRACUT_SYSTEMD" ] && reloadsysrootmountunit="yes"
+                    fi
+                    setup="yes"
+                fi
+            fi
+        elif [ -d /run/initramfs/overlayfs$pathspec ] &&
+            [ -d /run/initramfs/overlayfs$pathspec/../ovlwork ]; then
+            ln -s /run/initramfs/overlayfs$pathspec /run/overlayfs$opt
+            ln -s /run/initramfs/overlayfs$pathspec/../ovlwork /run/ovlwork$opt
+            if [ -z "$overlayfs" ]; then
+                overlayfs="yes"
+                [ -n "$DRACUT_SYSTEMD" ] && reloadsysrootmountunit="yes"
             fi
             setup="yes"
         fi
-        umount -l /run/initramfs/overlayfs || :
+    fi
+    if [ -n "$overlayfs" ]; then
+        modprobe overlay
+        if [ $? != 0 ]; then
+            echo "OverlayFS is not available; using temporary Device-mapper overlay." > /dev/kmsg
+            unset -v overlayfs setup
+            [ -n "$reloadsysrootmountunit" ] && unset -v reloadsysrootmountunit
+            sleep 5
+        fi
     fi
 
     if [ -z "$setup" -o -n "$readonly_overlay" ]; then
         if [ -n "$setup" ]; then
             warn "Using temporary overlay."
         elif [ -n "$devspec" -a -n "$pathspec" ]; then
-            warn "Unable to find persistent overlay; using temporary"
+            echo "Unable to find persistent overlay; using temporary." > /dev/kmsg
             sleep 5
         fi
-
-        dd if=/dev/null of=/overlay bs=1024 count=1 seek=$((overlay_size*1024)) 2> /dev/null
-        if [ -n "$setup" -a -n "$readonly_overlay" ]; then
-            RO_OVERLAY_LOOPDEV=$( losetup -f )
-            losetup $RO_OVERLAY_LOOPDEV /overlay
+        if [ -n "$overlayfs" ]; then
+            mkdir -m 0755 /run/overlayfs
+            mkdir -m 0755 /run/ovlwork
         else
-            losetup $OVERLAY_LOOPDEV /overlay
+            dd if=/dev/null of=/overlay bs=1024 count=1 seek=$((overlay_size*1024)) 2> /dev/null
+            if [ -n "$setup" -a -n "$readonly_overlay" ]; then
+                RO_OVERLAY_LOOPDEV=$( losetup -f )
+                losetup $RO_OVERLAY_LOOPDEV /overlay
+            else
+                losetup $OVERLAY_LOOPDEV /overlay
+            fi
         fi
     fi
 
     # set up the snapshot
     sz=$(blockdev --getsz $BASE_LOOPDEV)
-    if [ -n "$readonly_overlay" ]; then
-        echo 0 $sz snapshot $BASE_LOOPDEV $OVERLAY_LOOPDEV N 8 | dmsetup create --readonly live-ro
-        base="/dev/mapper/live-ro"
-        over=$RO_OVERLAY_LOOPDEV
-    else
-        base=$BASE_LOOPDEV
-        over=$OVERLAY_LOOPDEV
+    if [ -z "$overlayfs" ]; then
+        if [ -n "$readonly_overlay" ]; then
+            echo 0 $sz snapshot $BASE_LOOPDEV $OVERLAY_LOOPDEV P 8 | dmsetup create --readonly live-ro
+            base="/dev/mapper/live-ro"
+            over=$RO_OVERLAY_LOOPDEV
+        else
+            base=$BASE_LOOPDEV
+            over=$OVERLAY_LOOPDEV
+        fi
     fi
 
     if [ -n "$thin_snapshot" ]; then
         modprobe dm_thin_pool
-        mkdir /run/initramfs/thin-overlay
+        mkdir -m 0755 /run/initramfs/thin-overlay
 
         # In block units (512b)
         thin_data_sz=$(( $overlay_size * 1024 * 1024 / 512 ))
@@ -170,25 +217,17 @@ do_live_overlay() {
 
         # Create a snapshot of the base image
         echo 0 $sz thin /dev/mapper/live-overlay-pool 0 $base | dmsetup create live-rw
-    else
+    elif [ -z "$overlayfs" ]; then
         echo 0 $sz snapshot $base $over PO 8 | dmsetup create live-rw
     fi
 
     # Create a device that always points to a ro base image
     echo 0 $sz linear $BASE_LOOPDEV 0 | dmsetup create --readonly live-base
 }
 
-# live cd helper function
-do_live_from_base_loop() {
-    do_live_overlay
-}
-
 # we might have a genMinInstDelta delta file for anaconda to take advantage of
 if [ -e /run/initramfs/live/${live_dir}/osmin.img ]; then
     OSMINSQFS=/run/initramfs/live/${live_dir}/osmin.img
-fi
-
-if [ -n "$OSMINSQFS" ]; then
     # decompress the delta data
     dd if=$OSMINSQFS of=/run/initramfs/osmin.img 2> /dev/null
     OSMIN_SQUASHED_LOOPDEV=$( losetup -f )
@@ -204,13 +243,12 @@ fi
 if [ -e /run/initramfs/live/${live_dir}/${squash_image} ]; then
     SQUASHED="/run/initramfs/live/${live_dir}/${squash_image}"
 fi
-
-if [ -e "$SQUASHED" ] ; then
+if [ -e "$SQUASHED" ]; then
     if [ -n "$live_ram" ]; then
-        echo "Copying live image to RAM..."
-        echo "(this may take a few minutes)"
+        echo 'Copying live image to RAM...' > /dev/kmsg
+        echo ' (this may take a minute)' > /dev/kmsg
         dd if=$SQUASHED of=/run/initramfs/squashed.img bs=512 2> /dev/null
-        echo "Done copying live image to RAM."
+        echo 'Done copying live image to RAM.' > /dev/kmsg
         SQUASHED="/run/initramfs/squashed.img"
     fi
 
@@ -219,23 +257,23 @@ if [ -e "$SQUASHED" ] ; then
     mkdir -m 0755 -p /run/initramfs/squashfs
     mount -n -t squashfs -o ro $SQUASHED_LOOPDEV /run/initramfs/squashfs
 
-    if [ -f /run/initramfs/squashfs/LiveOS/ext3fs.img ]; then
-        FSIMG="/run/initramfs/squashfs/LiveOS/ext3fs.img"
-    elif [ -f /run/initramfs/squashfs/LiveOS/rootfs.img ]; then
+    if [ -f /run/initramfs/squashfs/LiveOS/rootfs.img ]; then
         FSIMG="/run/initramfs/squashfs/LiveOS/rootfs.img"
+    elif [ -f /run/initramfs/squashfs/LiveOS/ext3fs.img ]; then
+        FSIMG="/run/initramfs/squashfs/LiveOS/ext3fs.img"
     fi
 else
     # we might have an embedded fs image to use as rootfs (uncompressed live)
-    if [ -e /run/initramfs/live/${live_dir}/ext3fs.img ]; then
-        FSIMG="/run/initramfs/live/${live_dir}/ext3fs.img"
-    elif [ -e /run/initramfs/live/${live_dir}/rootfs.img ]; then
+    if [ -e /run/initramfs/live/${live_dir}/rootfs.img ]; then
         FSIMG="/run/initramfs/live/${live_dir}/rootfs.img"
+    elif [ -e /run/initramfs/live/${live_dir}/ext3fs.img ]; then
+        FSIMG="/run/initramfs/live/${live_dir}/ext3fs.img"
     fi
     if [ -n "$live_ram" ]; then
-        echo 'Copying live image to RAM...'
-        echo '(this may take a few minutes)'
+        echo 'Copying live image to RAM...' > /dev/kmsg
+        echo ' (this may take a minute or so)' > /dev/kmsg
         dd if=$FSIMG of=/run/initramfs/rootfs.img bs=512 2> /dev/null
-        echo 'Done copying live image to RAM.'
+        echo 'Done copying live image to RAM.' > /dev/kmsg
         FSIMG='/run/initramfs/rootfs.img'
     fi
 fi
@@ -245,44 +283,69 @@ if [ -n "$FSIMG" ] ; then
 
     if [ -n "$writable_fsimg" ] ; then
         # mount the provided filesystem read/write
-        echo "Unpacking live filesystem (may take some time)"
-        mkdir /run/initramfs/fsimg/
+        echo "Unpacking live filesystem (may take some time)" > /dev/kmsg
+        mkdir -m 0755 /run/initramfs/fsimg/
         if [ -n "$SQUASHED" ]; then
             cp -v $FSIMG /run/initramfs/fsimg/rootfs.img
         else
             unpack_archive $FSIMG /run/initramfs/fsimg/
         fi
         FSIMG=/run/initramfs/fsimg/rootfs.img
     fi
-    if [ -n "$writable_fsimg" ] || [ -z "$SQUASHED" -a -n "$live_ram" ] ||
+    if [ -n "$writable_fsimg" ] || [ -z "$SQUASHED" -a -n "$live_ram" -a -z "$overlayfs" ] ||
        [ "$overlay" = none -o "$overlay" = None -o "$overlay" = NONE ]; then
         losetup $BASE_LOOPDEV $FSIMG
         sz=$(blockdev --getsz $BASE_LOOPDEV)
         echo 0 $sz linear $BASE_LOOPDEV 0 | dmsetup create live-rw
     else
-        # mount the filesystem read-only and add a dm snapshot for writes
+        # Attach the filesystem read-only and add a DM snapshot or OverlayFS for writes.
         losetup -r $BASE_LOOPDEV $FSIMG
-        do_live_from_base_loop
+        do_live_overlay
     fi
 fi
 
-[ -e "$SQUASHED" ] && umount -l /run/initramfs/squashfs
+[ -e "$SQUASHED" ] && [ -z "$overlayfs" ] && umount -l /run/initramfs/squashfs
 
 if [ -b "$OSMIN_LOOPDEV" ]; then
     # set up the devicemapper snapshot device, which will merge
     # the normal live fs image, and the delta, into a minimzied fs image
-    echo "0 $( blockdev --getsz $BASE_LOOPDEV ) snapshot $BASE_LOOPDEV $OSMIN_LOOPDEV N 8" | dmsetup create --readonly live-osimg-min
+    echo "0 $sz snapshot $BASE_LOOPDEV $OSMIN_LOOPDEV P 8" | dmsetup create --readonly live-osimg-min
 fi
 
-ROOTFLAGS="$(getarg rootflags)"
-if [ -n "$ROOTFLAGS" ]; then
-    ROOTFLAGS="-o $ROOTFLAGS"
+if [ -n "$reloadsysrootmountunit" ]; then
+    > /xor_overlayfs
+    systemctl daemon-reload
 fi
 
-ln -s /dev/mapper/live-rw /dev/root
+ROOTFLAGS="$(getarg rootflags)"
 
-if [ -z "$DRACUT_SYSTEMD" ]; then
-    printf 'mount %s /dev/mapper/live-rw %s\n' "$ROOTFLAGS" "$NEWROOT" > $hookdir/mount/01-$$-live.sh
+if [ -n "$overlayfs" ]; then
+    mkdir -m 0755 /run/rootfsbase
+    if [ -n "$reset_overlay" ] && [ -L /run/overlayfs ]; then
+        ovlfs=$(readlink /run/overlayfs)
+        info "Resetting the OverlayFS overlay directory."
+        rm -r -- ${ovlfs}/* ${ovlfs}/.* >/dev/null 2>&1
+    fi
+    if [ -n "$readonly_overlay" ]; then
+        mkdir -m 0755 /run/rootfsbase-r
+        mount -r $FSIMG /run/rootfsbase-r
+        mount -t overlay LiveOS_rootfs-r -oro,lowerdir=/run/overlayfs-r:/run/rootfsbase-r /run/rootfsbase
+    else
+        mount -r $FSIMG /run/rootfsbase
+    fi
+    if [ -z "$DRACUT_SYSTEMD" ]; then
+        #FIXME What to link to /dev/root? Is it even needed?
+        printf 'mount -t overlay LiveOS_rootfs -o%s,%s %s\n' "$ROOTFLAGS" \
+        'lowerdir=/run/rootfsbase,upperdir=/run/overlayfs,workdir=/run/ovlwork' \
+        "$NEWROOT" > $hookdir/mount/01-$$-live.sh
+    fi
+else
+    ln -s /dev/mapper/live-rw /dev/root
+    if [ -z "$DRACUT_SYSTEMD" ]; then
+        [ -n "$ROOTFLAGS" ] && ROOTFLAGS="-o $ROOTFLAGS"
+        printf 'mount %s /dev/mapper/live-rw %s\n' "$ROOTFLAGS" "$NEWROOT" > $hookdir/mount/01-$$-live.sh
+    fi
+    ln -s /dev/mapper/live-base /run/rootfsbase
 fi
 
 need_shutdown

modules.d/90dmsquash-live/module-setup.sh

@@ -17,7 +17,7 @@ depends() {
 
 # called by dracut
 installkernel() {
-    instmods squashfs loop iso9660
+    instmods squashfs loop iso9660 overlay
 }
 
 # called by dracut

modules.d/90dmsquash-live/parse-dmsquash-live.sh

@@ -57,6 +57,6 @@ info "root was $liveroot, is now $root"
 # make sure that init doesn't complain
 [ -z "$root" ] && root="live"
 
-wait_for_dev -n /dev/mapper/live-rw
+wait_for_dev -n /run/rootfsbase
 
 return 0