add support for writing files to containers from cloud-init config

We are adding the support for "write_files" section in cloud-init config
for containers. Also the definition of envs is moved to the section
"runcmd" with lines like "- VAR=value". The rest of cloud-init config is
ignored.

Signed-off-by: Paul Gaiduk <paulg@zededa.com>

pkg/pillar/cmd/domainmgr/domainmgr.go

@@ -9,6 +9,7 @@
 package domainmgr
 
 import (
+	"bufio"
 	"encoding/base64"
 	"errors"
 	"flag"
@@ -21,6 +22,7 @@ import (
 	"time"
 
 	"github.com/containerd/cgroups"
+	cloudconfig "github.com/elotl/cloud-init/config"
 	"github.com/google/go-cmp/cmp"
 	zconfig "github.com/lf-edge/eve-api/go/config"
 	"github.com/lf-edge/eve/pkg/pillar/agentbase"
@@ -38,6 +40,7 @@ import (
 	"github.com/lf-edge/eve/pkg/pillar/sriov"
 	"github.com/lf-edge/eve/pkg/pillar/types"
 	"github.com/lf-edge/eve/pkg/pillar/utils"
+	fileutils "github.com/lf-edge/eve/pkg/pillar/utils/file"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	uuid "github.com/satori/go.uuid"
 	"github.com/sirupsen/logrus"
@@ -1404,6 +1407,66 @@ func doAssignIoAdaptersToDomain(ctx *domainContext, config types.DomainConfig,
 	return nil
 }
 
+func addFileFromCloudInit(file cloudconfig.File, rootPath string) error {
+	// transform file.Permission to os.FileMode
+	perm, err := strconv.ParseUint(file.RawFilePermissions, 8, 32)
+	if err != nil {
+		return err
+	}
+	mode := os.FileMode(perm)
+
+	writePath := filepath.Join(rootPath, file.Path)
+	contentBytes, err := cloudconfig.DecodeContent(file.Content, file.Encoding)
+	if err != nil {
+		return err
+	}
+	log.Tracef("Creating file %s with mode %s in %s\n", file.Path, mode, rootPath)
+	err = fileutils.WriteRename(writePath, contentBytes)
+	if err != nil {
+		return err
+	}
+	err = os.Chmod(writePath, mode)
+	if err != nil {
+		return err
+	}
+	if file.Owner != "" {
+		log.Warn("Changing owner of files written by cloud-init is not supported yet")
+	}
+
+	return nil
+}
+
+func getVersionFromMetaFile(path string) (uint64, error) {
+	var curCIVersion uint64 = 0 // for the case meta-data file doesn't exist
+
+	// read cloud init version from the meta-data file
+	metafile, err := os.Open(path)
+	if err != nil {
+		return 0, fmt.Errorf("Failed to open meta-data file: %s", err)
+	}
+	scanner := bufio.NewScanner(metafile)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.HasPrefix(line, "instance-id:") {
+			parts := strings.Split(line, "/")
+			if len(parts) >= 2 {
+				curCIVersion, err = strconv.ParseUint(parts[1], 10, 32)
+				if err != nil {
+					return 0, fmt.Errorf("Failed to parse cloud init version: %s", err.Error())
+				}
+				return curCIVersion, nil
+			}
+		}
+	}
+
+	// Check for scanner errors.
+	if err := scanner.Err(); err != nil {
+		return curCIVersion, fmt.Errorf("Reading the file failed: %s", err.Error())
+	}
+
+	return curCIVersion, errors.New("Version not found in meta-data file")
+}
+
 func doActivate(ctx *domainContext, config types.DomainConfig,
 	status *types.DomainStatus) {
 
@@ -1463,13 +1526,56 @@ func doActivate(ctx *domainContext, config types.DomainConfig,
 			// do nothing
 		case zconfig.Format_CONTAINER:
 			snapshotID := containerd.GetSnapshotID(ds.FileLocation)
-			if err := ctx.casClient.MountSnapshot(snapshotID, cas.GetRoofFsPath(ds.FileLocation)); err != nil {
+			rootPath := cas.GetRoofFsPath(ds.FileLocation)
+			if err := ctx.casClient.MountSnapshot(snapshotID, rootPath); err != nil {
 				err := fmt.Errorf("doActivate: Failed mount snapshot: %s for %s. Error %s",
 					snapshotID, config.UUIDandVersion.UUID, err)
 				log.Error(err.Error())
 				status.SetErrorNow(err.Error())
 				return
 			}
+
+			log.Warnf("Config: \n%+v\n", config)
+			log.Warnf("Status: \n%+v\n", *status)
+
+			metadataPath := filepath.Join(rootPath, "meta-data")
+
+			curCIVersion, err := getVersionFromMetaFile(metadataPath)
+			if err != nil {
+				log.Error("Failed to get cloud init version from meta-data file: ", err)
+				curCIVersion = 0 // make sure the cloud init config gets executed
+			}
+
+			newCIVersion, err := strconv.ParseUint(getCloudInitVersion(config), 10, 32)
+			if err != nil {
+				log.Error("Failed to parse cloud init version: ", err)
+				newCIVersion = curCIVersion + 1 // make sure the cloud init config gets executed
+			}
+
+			if curCIVersion < newCIVersion {
+				log.Notice("New cloud init config detected - applying")
+
+				// write meta-data file
+				versionString := fmt.Sprintf("instance-id: %s/%s\n", config.UUIDandVersion.UUID.String(), getCloudInitVersion(config))
+				err = fileutils.WriteRename(metadataPath, []byte(versionString))
+				if err != nil {
+					err := fmt.Errorf("doActivate: Failed to write cloud-init metadata file. Error %s", err)
+					log.Error(err.Error())
+					status.SetErrorNow(err.Error())
+					return
+				}
+
+				// apply cloud init config
+				for _, writableFile := range status.WritableFiles {
+					err := addFileFromCloudInit(writableFile, rootPath)
+					if err != nil {
+						err := fmt.Errorf("doActivate: Failed to apply cloud-init config. Error %s", err)
+						log.Error(err.Error())
+						status.SetErrorNow(err.Error())
+						return
+					}
+				}
+			}
 		default:
 			// assume everything else to be disk formats
 			format, err := utils.GetVolumeFormat(log, ds.FileLocation)
@@ -1889,20 +1995,32 @@ func configToStatus(ctx *domainContext, config types.DomainConfig,
 	//clean environment variables
 	status.EnvVariables = nil
 
+	// Fetch cloud-init userdata
 	if config.IsCipher || config.CloudInitUserData != nil {
 		ciStr, err := fetchCloudInit(ctx, config)
 		if err != nil {
 			return fmt.Errorf("failed to fetch cloud-init userdata: %s",
 				err)
 		}
-		if status.OCIConfigDir != "" {
-			envList, err := parseEnvVariablesFromCloudInit(ciStr)
+		if status.OCIConfigDir != "" { // If AppInstance is a container, we need to parse cloud-init config and apply the supported parts
+			cc, err := cloudconfig.NewCloudConfig(ciStr)
+			if err != nil {
+				return fmt.Errorf("failed to unmarshal cloud-init userdata: %s",
+					err)
+			}
+			if err := cloudconfig.AssertStructValid(cc); err != nil {
+				return fmt.Errorf("cloud-init userdata is not valid: %s",
+					err)
+			}
+			status.WritableFiles = cc.WriteFiles
+
+			envList, err := parseEnvVariablesFromCloudInit(cc.RunCmd)
 			if err != nil {
 				return fmt.Errorf("failed to parse environment variable from cloud-init userdata: %s",
 					err)
 			}
 			status.EnvVariables = envList
-		} else {
+		} else { // If AppInstance is a VM, we need to create a cloud-init ISO
 			switch config.MetaDataType {
 			case types.MetaDataDrive, types.MetaDataDriveMultipart:
 				ds, err := createCloudInitISO(ctx, config, ciStr)
@@ -2515,11 +2633,10 @@ func fetchCloudInit(ctx *domainContext,
 // Example:
 // Key1=Val1
 // Key2=Val2 ...
-func parseEnvVariablesFromCloudInit(ciStr string) (map[string]string, error) {
+func parseEnvVariablesFromCloudInit(envPairs []string) (map[string]string, error) {
 
 	envList := make(map[string]string, 0)
-	list := strings.Split(ciStr, "\n")
-	for _, v := range list {
+	for _, v := range envPairs {
 		pair := strings.SplitN(v, "=", 2)
 		if len(pair) != 2 {
 			errStr := fmt.Sprintf("Variable \"%s\" not defined properly\nKey value pair should be delimited by \"=\"", pair[0])

pkg/pillar/cmd/domainmgr/domainmgr_test.go

@@ -17,6 +17,7 @@ import (
 	"strings"
 	"testing"
 
+	cloudconfig "github.com/elotl/cloud-init/config"
 	"github.com/lf-edge/eve/pkg/pillar/base"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
@@ -104,7 +105,12 @@ func decodeAndParseEnvVariablesFromCloudInit(ciStr string) (map[string]string, e
 		return nil, fmt.Errorf("base64 decode failed %s", err)
 	}
 
-	return parseEnvVariablesFromCloudInit(string(ud))
+	cc, err := cloudconfig.NewCloudConfig(string(ud))
+	if err != nil {
+		return nil, err
+	}
+
+	return parseEnvVariablesFromCloudInit(cc.RunCmd)
 }
 
 // Definitions of various cloud-init multi-part messages

pkg/pillar/go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/cshari-zededa/eve-tpm2-tools v0.0.4
 	github.com/digitalocean/go-qemu v0.0.0-20220826173844-d5f5e3ceed89
 	github.com/docker/docker v20.10.24+incompatible
+	github.com/elotl/cloud-init v1.14.2
 	github.com/eriknordmark/ipinfo v0.0.0-20230728132417-2d8f4da903d7
 	github.com/fsnotify/fsnotify v1.5.1
 	github.com/go-chi/chi/v5 v5.0.10
@@ -71,6 +72,7 @@ require (
 	github.com/containerd/stargz-snapshotter/estargz v0.11.4 // indirect
 	github.com/containerd/ttrpc v1.1.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
+	github.com/coreos/yaml v0.0.0-20141224210557-6b16a5714269 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/digitalocean/go-libvirt v0.0.0-20221020193630-0d0212f5ead2 // indirect
 	github.com/docker/cli v20.10.17+incompatible // indirect

pkg/pillar/go.sum

@@ -514,6 +514,8 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV
 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/coreos/yaml v0.0.0-20141224210557-6b16a5714269 h1:/1sjrpK5Mb6IwyFOKd+u7321tXfNAsj0Ci8CivZmSlo=
+github.com/coreos/yaml v0.0.0-20141224210557-6b16a5714269/go.mod h1:Bl1D/T9QJhVdu6eFoLrGxN90+admDLGaLz2HXH/VzDc=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
@@ -619,6 +621,8 @@ github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkg
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
 github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
+github.com/elotl/cloud-init v1.14.2 h1:MsePTg+R45gxo8Y5Rmms/XwRHh61wdvD+4MDCisw7yo=
+github.com/elotl/cloud-init v1.14.2/go.mod h1:oCbqtZwJTJn+m5kxlJPcAJ5BjOLQuDm0Tw9IzT11XL8=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -1117,8 +1121,6 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
 github.com/lf-edge/edge-containers v0.0.0-20221025050409-93c34bebadd2 h1:ckxNk8MEdATh8ZsArR7puG9PI5izRzCT+/TE9dvuAwM=
 github.com/lf-edge/edge-containers v0.0.0-20221025050409-93c34bebadd2/go.mod h1:eA41YxPbZRVvewIYRzmqDB1PeLQXxCy9WQEc3AVCsPI=
-github.com/lf-edge/eve-api/go v0.0.0-20230917094129-590dad30fe13 h1:10Bwbfl1w63u4t/+7t3XDBb20A+WPCBsmMTeYkW89B8=
-github.com/lf-edge/eve-api/go v0.0.0-20230917094129-590dad30fe13/go.mod h1:6XqpOM8p1HsluNIGw2ihYPYsaAisQ5CuJpbIKHXQo5w=
 github.com/lf-edge/eve-api/go v0.0.0-20231011200019-cb3cb1275e0d h1:PVKqYtPsH5BAIYfOaKej/+lc7+GKcFZBGnzbS6JWbrE=
 github.com/lf-edge/eve-api/go v0.0.0-20231011200019-cb3cb1275e0d/go.mod h1:6XqpOM8p1HsluNIGw2ihYPYsaAisQ5CuJpbIKHXQo5w=
 github.com/lf-edge/eve-libs v0.0.0-20230921141205-94d6f6b65597 h1:/UGYRj5tdRw5m3+VjZtTx1RVgphQbthfY/Gu5W7qb5o=

pkg/pillar/types/domainmgrtypes.go

@@ -13,6 +13,7 @@ import (
 
 	uuid "github.com/satori/go.uuid"
 
+	cloudconfig "github.com/elotl/cloud-init/config"
 	"github.com/google/go-cmp/cmp"
 	zconfig "github.com/lf-edge/eve-api/go/config"
 	"github.com/lf-edge/eve/pkg/pillar/base"
@@ -278,9 +279,10 @@ type DomainStatus struct {
 	ConfigFailed   bool
 	BootFailed     bool
 	AdaptersFailed bool
-	OCIConfigDir   string            // folder holding an OCI Image config for this domain (empty string means no config)
-	EnvVariables   map[string]string // List of environment variables to be set in container
-	VmConfig                         // From DomainConfig
+	OCIConfigDir   string             // folder holding an OCI Image config for this domain (empty string means no config)
+	EnvVariables   map[string]string  // List of environment variables to be set in container
+	WritableFiles  []cloudconfig.File // List of files from CloudInit scripts to be created in container
+	VmConfig                          // From DomainConfig
 	Service        bool
 }