Add filesystem cleanExcept directive to preserve wanted files

The wipeFilesystem directive causes all state to be lost. A more
fine-grained mechanism is needed to clean a filesystem from previous
unwanted state while allowing for some files or directories to be kept.

Add a new cleanExcept directive that when specified will remove all
directories and files on the filesystem except those that match a list
of regular expressions.

config/v3_4_experimental/schema/ignition.json

@@ -349,6 +349,12 @@
             "wipeFilesystem": {
               "type": ["boolean", "null"]
             },
+            "cleanExcept": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            },
             "label": {
               "type": ["string", "null"]
             },

config/v3_4_experimental/types/schema.go

@@ -2,6 +2,8 @@ package types
 
 // generated by "schematyper --package=types config/v3_4_experimental/schema/ignition.json -o config/v3_4_experimental/types/schema.go --root-type=Config" -- DO NOT EDIT
 
+type CleanExceptItem string
+
 type Clevis struct {
 	Custom    ClevisCustom `json:"custom,omitempty"`
 	Tang      []Tang       `json:"tang,omitempty"`
@@ -57,6 +59,7 @@ type FileEmbedded1 struct {
 }
 
 type Filesystem struct {
+	CleanExcept    []CleanExceptItem  `json:"cleanExcept,omitempty"`
 	Device         string             `json:"device"`
 	Format         *string            `json:"format,omitempty"`
 	Label          *string            `json:"label,omitempty"`

docs/configuration-v3_4_experimental.md

@@ -71,6 +71,7 @@ The Ignition configuration is a JSON document conforming to the following specif
     * **format** (string): the filesystem format (ext4, btrfs, xfs, vfat, swap, or none).
     * **_path_** (string): the mount-point of the filesystem while Ignition is running relative to where the root filesystem will be mounted. This is not necessarily the same as where it should be mounted in the real root, but it is encouraged to make it the same.
     * **_wipeFilesystem_** (boolean): whether or not to wipe the device before filesystem creation, see [the documentation on filesystems](operator-notes.md#filesystem-reuse-semantics) for more information. Defaults to false.
+    * **_cleanExcept_** (list of strings): delete files and directories not matching one of the regular expressions of the list. The regular expressions must not include calculation over path dividers (`/`). If a directory matches, its contents are excluded, too. If you specify `"/"` which refers to the filesystems top directory, nothing will be cleaned. An empty list also means nothing is cleaned, use **wipeFilesystem** instead to discard the whole filesystem. The `cleanExcept` directive is useful to preserve application or system state, for example, on the root filesystem something like `/etc/ssh/ssh_host_.*` can preserve SSH host keys, `/var/log` preserves logs, or something like `/var/lib/docker`, `/var/lib/containerd` can preserve container state and images (Note: Do not use this for `/etc/machine-id` but set it through the kernel command line parameter `systemd.machine_id=`, otherwise the Ignition systemd presets are not evaluated.).
     * **_label_** (string): the label of the filesystem.
     * **_uuid_** (string): the uuid of the filesystem.
     * **_options_** (list of strings): any additional options to be passed to the format-specific mkfs utility.

internal/exec/stages/disks/filesystems.go

@@ -21,9 +21,15 @@ package disks
 import (
 	"errors"
 	"fmt"
+	iofs "io/fs"
+	"io/ioutil"
+	"os"
 	"os/exec"
+	"path/filepath"
+	"regexp"
 	"runtime"
 	"strings"
+	"syscall"
 
 	cutil "github.com/coreos/ignition/v2/config/util"
 	"github.com/coreos/ignition/v2/config/v3_4_experimental/types"
@@ -88,6 +94,102 @@ func (s stage) createFilesystems(config types.Config) error {
 	return nil
 }
 
+func (s stage) cleanFilesystemExcept(fs types.Filesystem) error {
+        s.Logger.Info("filesystem at %q needs to be cleaned, preserving only %q", fs.Device, fs.CleanExcept)
+        var cleanExceptRegex []*regexp.Regexp
+        var keepButDontSkipRegex []*regexp.Regexp
+        for _, regex := range fs.CleanExcept {
+                // Remove final "/" in case it was passed to specify directories
+                pathRegex := strings.TrimSuffix(string(regex), "/")
+                if len(pathRegex) == 0 {
+                        // Keeping the whole top directory is a no-op
+                        return nil
+                }
+                regexKeep, err := regexp.Compile(pathRegex)
+                if err != nil {
+                        return err
+                }
+                cleanExceptRegex = append(cleanExceptRegex, regexKeep)
+                cleanExceptRegex = append(cleanExceptRegex, regexKeep)
+                // Assemble a list of parent directory regular expressions,
+                // "/" is not an allowed part of a regular expression and things
+                // will break if it's not used as literal without repetitions/omissions
+                // (probably an error is reported because the split string won't be valid)
+                parts := strings.Split(pathRegex, "/")
+                for i := len(parts) - 1; i > 1; i-- {
+                        partsParent := parts[0:i]
+                        regexForParent, err := regexp.Compile("/" + filepath.Join(partsParent...))
+                        if err != nil {
+                                return fmt.Errorf("split regex not valid, you must not use '/' as part of a regular expression:%v", err)
+                        }
+                        keepButDontSkipRegex = append(keepButDontSkipRegex, regexForParent)
+                }
+        }
+        mnt, err := ioutil.TempDir("", "clean-filesystem-except")
+        if err != nil {
+                return fmt.Errorf("failed to create temp directory: %v", err)
+        }
+        // Make sure mnt does not end with a "/" because we use it to cut the path prefix
+        mnt = strings.TrimSuffix(mnt, "/")
+        defer os.Remove(mnt)
+        dev := string(fs.Device)
+        format := string(*fs.Format)
+        if err := syscall.Mount(dev, mnt, format, 0, ""); err != nil {
+                return err
+        }
+        defer s.Logger.LogOp(
+                func() error { return syscall.Unmount(mnt, 0) },
+                "unmounting %q at %q", dev, mnt,
+        )
+        return filepath.WalkDir(mnt, func(path string, d iofs.DirEntry, err error) error {
+                if err != nil {
+                        if errors.Is(err, os.ErrNotExist) {
+                                return fmt.Errorf("hit a deleted file (programming error): %v", err)
+                        }
+                        return err
+                }
+                // Assumption: The "path" we get is already absolute and for directories it does not end with a "/"
+                matchPath := strings.Replace(path, mnt, "", 1)
+                if matchPath == "" {
+                        // Skip top directory (first function call)
+                        return nil
+                }
+                match := false
+                for _, regexKeep := range cleanExceptRegex {
+                        if loc := regexKeep.FindStringIndex(matchPath); loc != nil && loc[0] == 0 && loc[1] == len(matchPath) {
+                                match = true
+                                break
+                        }
+                }
+                matchkeepButDontSkip := false
+                for _, regexKeepButDontSkip := range keepButDontSkipRegex {
+                        if loc := regexKeepButDontSkip.FindStringIndex(matchPath); loc != nil && loc[0] == 0 && loc[1] == len(matchPath) {
+                                matchkeepButDontSkip = true
+                                break
+                        }
+                }
+                if match && d.IsDir() {
+                        return iofs.SkipDir
+                }
+                if matchkeepButDontSkip && d.IsDir() {
+                        return nil
+                }
+                if match {
+                        // Keep matched file
+                        return nil
+                }
+                removeErr := os.RemoveAll(path)
+                if removeErr != nil {
+                        return removeErr
+                }
+                if d.IsDir() {
+                        // We removed the directory and the contents already, and can't enter it anymore
+                        return iofs.SkipDir
+                }
+                return nil
+        })
+}
+
 func (s stage) createFilesystem(fs types.Filesystem) error {
 	if fs.Format == nil {
 		return nil
@@ -130,6 +232,9 @@ func (s stage) createFilesystem(fs types.Filesystem) error {
 			(fs.Label == nil || info.Label == *fs.Label) &&
 			(fs.UUID == nil || canonicalizeFilesystemUUID(info.Type, info.UUID) == canonicalizeFilesystemUUID(fileSystemFormat, *fs.UUID)) {
 			s.Logger.Info("filesystem at %q is already correctly formatted. Skipping mkfs...", fs.Device)
+			if len(fs.CleanExcept) > 0 {
+				return s.cleanFilesystemExcept(fs)
+			}
 			return nil
 		} else if info.Type != "" {
 			s.Logger.Err("filesystem at %q is not of the correct type, label, or UUID (found %s, %q, %s) and a filesystem wipe was not requested", fs.Device, info.Type, info.Label, info.UUID)