Add persistence mechanism between stages; write result report to provisioned system

internal/distro/distro.go

@@ -27,8 +27,9 @@ var (
 	diskByLabelDir    = "/dev/disk/by-label"
 	diskByPartUUIDDir = "/dev/disk/by-partuuid"
 
-	// File paths
+	// initrd file paths
 	kernelCmdlinePath = "/proc/cmdline"
+	bootIDPath        = "/proc/sys/kernel/random/boot_id"
 	// initramfs directory containing distro-provided base config
 	systemConfigDir = "/usr/lib/ignition"
 
@@ -74,15 +75,17 @@ var (
 	// ".ssh/authorized_keys" ("false").
 	writeAuthorizedKeysFragment = "true"
 
-	luksInitramfsKeyFilePath = "/run/ignition/luks-keyfiles/"
-	luksRealRootKeyFilePath  = "/etc/luks/"
+	// Special file paths in the real root
+	luksRealRootKeyFilePath = "/etc/luks/"
+	resultFilePath          = "/var/lib/ignition/result.json"
 )
 
 func DiskByIDDir() string       { return diskByIDDir }
 func DiskByLabelDir() string    { return diskByLabelDir }
 func DiskByPartUUIDDir() string { return diskByPartUUIDDir }
 
 func KernelCmdlinePath() string { return kernelCmdlinePath }
+func BootIDPath() string        { return bootIDPath }
 func SystemConfigDir() string   { return fromEnv("SYSTEM_CONFIG_DIR", systemConfigDir) }
 
 func GroupaddCmd() string { return groupaddCmd }
@@ -113,8 +116,8 @@ func CryptsetupCmd() string { return cryptsetupCmd }
 
 func KargsCmd() string { return kargsCmd }
 
-func LuksInitramfsKeyFilePath() string { return luksInitramfsKeyFilePath }
-func LuksRealRootKeyFilePath() string  { return luksRealRootKeyFilePath }
+func LuksRealRootKeyFilePath() string { return luksRealRootKeyFilePath }
+func ResultFilePath() string          { return resultFilePath }
 
 func SelinuxRelabel() bool  { return bakedStringToBool(selinuxRelabel) && !BlackboxTesting() }
 func BlackboxTesting() bool { return bakedStringToBool(blackboxTesting) }

internal/exec/engine.go

@@ -40,6 +40,7 @@ import (
 	"github.com/coreos/ignition/v2/internal/providers/cmdline"
 	"github.com/coreos/ignition/v2/internal/providers/system"
 	"github.com/coreos/ignition/v2/internal/resource"
+	"github.com/coreos/ignition/v2/internal/state"
 	"github.com/coreos/ignition/v2/internal/util"
 
 	"github.com/coreos/vcontext/report"
@@ -69,13 +70,7 @@ type Engine struct {
 	Root           string
 	PlatformConfig platform.Config
 	Fetcher        *resource.Fetcher
-	fetchedConfigs []fetchedConfig
-}
-
-type fetchedConfig struct {
-	kind       string
-	source     string
-	referenced bool
+	State          *state.State
 }
 
 // Run executes the stage of the given name. It returns true if the stage
@@ -93,10 +88,10 @@ func (e Engine) Run(stageName string) error {
 		e.Logger.Crit("failed to acquire system base config: %v", err)
 		return err
 	} else if err == nil {
-		e.fetchedConfigs = append(e.fetchedConfigs, fetchedConfig{
-			kind:       "base",
-			source:     "system",
-			referenced: false,
+		e.State.FetchedConfigs = append(e.State.FetchedConfigs, state.FetchedConfig{
+			Kind:       "base",
+			Source:     "system",
+			Referenced: false,
 		})
 	}
 
@@ -138,7 +133,7 @@ func (e Engine) Run(stageName string) error {
 	defer e.Logger.PopPrefix()
 
 	fullConfig := latest.Merge(baseConfig, latest.Merge(systemBaseConfig, cfg))
-	err = stages.Get(stageName).Create(e.Logger, e.Root, *e.Fetcher).Run(fullConfig)
+	err = stages.Get(stageName).Create(e.Logger, e.Root, *e.Fetcher, e.State).Run(fullConfig)
 	if err == resource.ErrNeedNet && stageName == "fetch-offline" {
 		err = e.signalNeedNet()
 		if err != nil {
@@ -164,18 +159,18 @@ func (e Engine) Run(stageName string) error {
 
 // logStructuredJournalEntry logs information related to
 // a user/base config into the systemd journal log.
-func logStructuredJournalEntry(cfgInfo fetchedConfig) error {
+func logStructuredJournalEntry(cfgInfo state.FetchedConfig) error {
 	ignitionInfo := map[string]string{
-		"IGNITION_CONFIG_TYPE":       cfgInfo.kind,
-		"IGNITION_CONFIG_SRC":        cfgInfo.source,
-		"IGNITION_CONFIG_REFERENCED": strconv.FormatBool(cfgInfo.referenced),
+		"IGNITION_CONFIG_TYPE":       cfgInfo.Kind,
+		"IGNITION_CONFIG_SRC":        cfgInfo.Source,
+		"IGNITION_CONFIG_REFERENCED": strconv.FormatBool(cfgInfo.Referenced),
 		"MESSAGE_ID":                 ignitionFetchedConfigMsgId,
 	}
 	referenced := ""
-	if cfgInfo.referenced {
+	if cfgInfo.Referenced {
 		referenced = "referenced "
 	}
-	msg := fmt.Sprintf("fetched %s%s config from %q", referenced, cfgInfo.kind, cfgInfo.source)
+	msg := fmt.Sprintf("fetched %s%s config from %q", referenced, cfgInfo.Kind, cfgInfo.Source)
 	if err := journal.Send(msg, journal.PriInfo, ignitionInfo); err != nil {
 		return err
 	}
@@ -193,7 +188,7 @@ func (e *Engine) acquireConfig(stageName string) (cfg types.Config, err error) {
 
 		// if we've successfully fetched and cached the configs, log about them
 		if err == nil {
-			for _, cfgInfo := range e.fetchedConfigs {
+			for _, cfgInfo := range e.State.FetchedConfigs {
 				if logerr := logStructuredJournalEntry(cfgInfo); logerr != nil {
 					e.Logger.Info("failed to log systemd journal entry: %v", logerr)
 				}
@@ -324,10 +319,10 @@ func (e *Engine) fetchProviderConfig() (types.Config, error) {
 		return types.Config{}, err
 	}
 
-	e.fetchedConfigs = append(e.fetchedConfigs, fetchedConfig{
-		kind:       "user",
-		source:     providerKey,
-		referenced: false,
+	e.State.FetchedConfigs = append(e.State.FetchedConfigs, state.FetchedConfig{
+		Kind:       "user",
+		Source:     providerKey,
+		Referenced: false,
 	})
 
 	// Replace the HTTP client in the fetcher to be configured with the
@@ -429,10 +424,10 @@ func (e *Engine) fetchReferencedConfig(cfgRef types.Resource) (types.Config, err
 		e.Logger.Debug("fetched referenced config from data url with SHA512: %s", hex.EncodeToString(hash[:]))
 	}
 
-	e.fetchedConfigs = append(e.fetchedConfigs, fetchedConfig{
-		kind:       "user",
-		source:     u.Path,
-		referenced: true,
+	e.State.FetchedConfigs = append(e.State.FetchedConfigs, state.FetchedConfig{
+		Kind:       "user",
+		Source:     u.Path,
+		Referenced: true,
 	})
 
 	if err := util.AssertValid(cfgRef.Verification, rawCfg); err != nil {

internal/exec/stages/disks/disks.go

@@ -28,6 +28,7 @@ import (
 	"github.com/coreos/ignition/v2/internal/exec/util"
 	"github.com/coreos/ignition/v2/internal/log"
 	"github.com/coreos/ignition/v2/internal/resource"
+	"github.com/coreos/ignition/v2/internal/state"
 	"github.com/coreos/ignition/v2/internal/systemd"
 )
 
@@ -41,12 +42,13 @@ func init() {
 
 type creator struct{}
 
-func (creator) Create(logger *log.Logger, root string, f resource.Fetcher) stages.Stage {
+func (creator) Create(logger *log.Logger, root string, f resource.Fetcher, state *state.State) stages.Stage {
 	return &stage{
 		Util: util.Util{
 			DestDir: root,
 			Logger:  logger,
 			Fetcher: f,
+			State:   state,
 		},
 	}
 }

internal/exec/stages/disks/luks.go

@@ -25,14 +25,15 @@ import (
 	"os"
 	"os/exec"
 	"path"
-	"path/filepath"
 	"strings"
 
 	"github.com/coreos/ignition/v2/config/util"
 	"github.com/coreos/ignition/v2/config/v3_4_experimental/types"
 	"github.com/coreos/ignition/v2/internal/distro"
 	execUtil "github.com/coreos/ignition/v2/internal/exec/util"
 	"github.com/coreos/ignition/v2/internal/resource"
+
+	"github.com/vincent-petithory/dataurl"
 )
 
 var (
@@ -104,22 +105,26 @@ func (s *stage) createLuks(config types.Config) error {
 		return err
 	}
 
+	s.State.LuksPersistKeyFiles = make(map[string]string)
+
 	for _, luks := range config.Storage.Luks {
 		// TODO: allow Ignition generated KeyFiles for
 		// non-clevis devices that can be persisted.
 		// TODO: create devices in parallel.
 		// track whether Ignition creates the KeyFile
 		// so that it can be removed
 		var ignitionCreatedKeyFile bool
-		// create keyfile inside of tmpfs, it will be copied to the
-		// sysroot by the files stage
-		if err := os.MkdirAll(distro.LuksInitramfsKeyFilePath(), 0700); err != nil {
-			return fmt.Errorf("creating directory for keyfile: %v", err)
+		// create keyfile, remove on the way out
+		keyFile, err := ioutil.TempFile("", "ignition-luks-")
+		if err != nil {
+			return fmt.Errorf("creating keyfile: %w", err)
 		}
-		keyFilePath := filepath.Join(distro.LuksInitramfsKeyFilePath(), luks.Name)
+		keyFilePath := keyFile.Name()
+		keyFile.Close()
+		defer os.Remove(keyFilePath)
 		devAlias := execUtil.DeviceAlias(*luks.Device)
 		if util.NilOrEmpty(luks.KeyFile.Source) {
-			// create a keyfile
+			// generate keyfile contents
 			key, err := randHex(4096)
 			if err != nil {
 				return fmt.Errorf("generating keyfile: %v", err)
@@ -316,17 +321,21 @@ func (s *stage) createLuks(config types.Config) error {
 			}
 		}
 
-		// assume the user does not want a key file & remove it for clevis based devices
 		if ignitionCreatedKeyFile && luks.Clevis.IsPresent() {
+			// assume the user does not want the generated key & remove it
 			if _, err := s.Logger.LogCmd(
 				exec.Command(distro.CryptsetupCmd(), "luksRemoveKey", devAlias, keyFilePath),
 				"removing key file for %v", luks.Name,
 			); err != nil {
 				return fmt.Errorf("removing key file from luks device: %v", err)
 			}
-			if err := os.Remove(keyFilePath); err != nil {
-				return fmt.Errorf("removing key file: %v", err)
+		} else {
+			// store the key to be persisted into the real root
+			key, err := ioutil.ReadFile(keyFilePath)
+			if err != nil {
+				return fmt.Errorf("failed to read keyfile %q: %w", keyFilePath, err)
 			}
+			s.State.LuksPersistKeyFiles[luks.Name] = dataurl.EncodeBytes(key)
 		}
 	}
 

internal/exec/stages/fetch/fetch.go

@@ -24,6 +24,7 @@ import (
 	"github.com/coreos/ignition/v2/internal/exec/util"
 	"github.com/coreos/ignition/v2/internal/log"
 	"github.com/coreos/ignition/v2/internal/resource"
+	"github.com/coreos/ignition/v2/internal/state"
 )
 
 const (
@@ -36,11 +37,12 @@ func init() {
 
 type creator struct{}
 
-func (creator) Create(logger *log.Logger, root string, _ resource.Fetcher) stages.Stage {
+func (creator) Create(logger *log.Logger, root string, _ resource.Fetcher, state *state.State) stages.Stage {
 	return &stage{
 		Util: util.Util{
 			DestDir: root,
 			Logger:  logger,
+			State:   state,
 		},
 	}
 }

internal/exec/stages/fetch_offline/fetch-offline.go

@@ -28,6 +28,7 @@ import (
 	executil "github.com/coreos/ignition/v2/internal/exec/util"
 	"github.com/coreos/ignition/v2/internal/log"
 	"github.com/coreos/ignition/v2/internal/resource"
+	"github.com/coreos/ignition/v2/internal/state"
 	"github.com/coreos/ignition/v2/internal/util"
 )
 
@@ -41,11 +42,12 @@ func init() {
 
 type creator struct{}
 
-func (creator) Create(logger *log.Logger, root string, _ resource.Fetcher) stages.Stage {
+func (creator) Create(logger *log.Logger, root string, _ resource.Fetcher, state *state.State) stages.Stage {
 	return &stage{
 		Util: executil.Util{
 			DestDir: root,
 			Logger:  logger,
+			State:   state,
 		},
 	}
 }

internal/exec/stages/files/files.go

@@ -25,6 +25,7 @@ import (
 	"github.com/coreos/ignition/v2/internal/exec/util"
 	"github.com/coreos/ignition/v2/internal/log"
 	"github.com/coreos/ignition/v2/internal/resource"
+	"github.com/coreos/ignition/v2/internal/state"
 )
 
 const (
@@ -41,12 +42,13 @@ func init() {
 
 type creator struct{}
 
-func (creator) Create(logger *log.Logger, root string, f resource.Fetcher) stages.Stage {
+func (creator) Create(logger *log.Logger, root string, f resource.Fetcher, state *state.State) stages.Stage {
 	return &stage{
 		Util: util.Util{
 			DestDir: root,
 			Logger:  logger,
 			Fetcher: f,
+			State:   state,
 		},
 	}
 }
@@ -85,6 +87,10 @@ func (s stage) Run(config types.Config) error {
 		return fmt.Errorf("creating crypttab entries: %v", err)
 	}
 
+	if err := s.createResultFile(); err != nil {
+		return fmt.Errorf("creating result file: %v", err)
+	}
+
 	if err := s.relabelFiles(); err != nil {
 		return fmt.Errorf("failed to handle relabeling: %v", err)
 	}