WIP: kola: add run-upgrade command

This adds a new `run-upgrade` command focused on running upgrade tests.
It also adds a single test in that testsuite: `fcos.upgrade.basic`.

To run this test, one can do:

```
kola run-upgrade -v \
        --cosa-build /path/to/meta.json \
        --qemu-image /path/to/starting-image.qcow2
```

You can tell kola to automatically detect the parent image to start
from:

```
kola run-upgrade -v \
        --cosa-build /path/to/meta.json \
        --find-parent-image
```

For FCOS, this will fetch the metadata for the latest release for the
target stream. On AWS, it will use the AMI from there as the starting
image. On the QEMU platform, it will download the QEMU image locally
(with signature verification). The code is extensible to add support for
RHCOS and other target platforms.

Why make it a separate command from `run`? Multiple reasons:
1. As shown above, it's about multiple artifacts, not just the system
   under test. By contrast, `run` is largely about using a single
   artifact input. For example, on AWS, `--aws-ami` points to the
   *starting* image, and `--cosa-build` points to the target upgrade.
2. It's more expensive than other tests. To make it truly cross-platform
   and self-contained, it works by pushing the OSTree content to the
   node and serving it from there to itself. Therefore, it's not a test
   that developers would necessarily be interested in running locally
   very often (though it's definitely adapted for local tests too when
   needed).
3. Unlike `run`, it has some special semantics like
   `--find-parent-image` to make it easier to use.

Now, this is only part of the FCOS upgrade testing story. Here's roughly
how I see this all fit together:
1. The FCOS pipeline runs `kola run-upgrade -p qemu` and possibly
   `kola run-upgrade -p aws` after the basic `kola run` tests have
   passed.
2. Once the build is clean and pushed out to S3, its content will be
   imported into the annex/compose repo.
3. Once there, we can do more realistic tests by targeting the annex
   repo and a dedicated Cincinnati. For example, we can have canary
   nodes following those updates that started from various previous
   releases to catch any state-dependent issues. Another more explicit
   approach is a test that starts those nodes at the select releases and
   gate new releases on that test.

Essentially, the main advantage of this test is that we can do some
upgrade testing *before* pushing out any bits at all to S3. The major
bug category this is intended to catch are state-dependent ones (i.e.
anything that *isn't* captured by the OSTree commit).

However, it does also exercise many of the major parts of the update
system (zincati, rpm-ostree, ostree, libcurl).  Though it's clearly not
a replacement for more realistic e2e tests downstream.

cmd/kola/kola.go

@@ -17,19 +17,26 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
 	"path/filepath"
 	"sort"
+	"strings"
 	"text/tabwriter"
 
 	"github.com/coreos/pkg/capnslog"
+	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 
 	"github.com/coreos/mantle/cli"
+	"github.com/coreos/mantle/cosa"
+	"github.com/coreos/mantle/fcos"
 	"github.com/coreos/mantle/kola"
 	"github.com/coreos/mantle/kola/register"
+	"github.com/coreos/mantle/sdk"
+	"github.com/coreos/mantle/util"
 
 	// register OS test suite
 	_ "github.com/coreos/mantle/kola/registry"
@@ -57,6 +64,15 @@ will be ignored.
 		PreRun: preRun,
 	}
 
+	cmdRunUpgrade = &cobra.Command{
+		Use:     "run-upgrade [glob pattern...]",
+		Short:   "Run upgrade kola tests",
+		Long:    `Run all upgrade kola tests (default) or related groups.`,
+		Run:     runRunUpgrade,
+		PreRun:  preRunUpgrade,
+		PostRun: postRunUpgrade,
+	}
+
 	cmdList = &cobra.Command{
 		Use:   "list",
 		Short: "List kola test names",
@@ -73,18 +89,25 @@ This can be useful for e.g. serving locally built OSTree repos to qemu.
 		Run: runHttpServer,
 	}
 
-	listJSON bool
-	httpPort int
+	listJSON           bool
+	httpPort           int
+	findParentImage    bool
+	qemuImageDir       string
+	qemuImageDirIsTemp bool
 )
 
 func init() {
 	root.AddCommand(cmdRun)
-	root.AddCommand(cmdList)
 
+	root.AddCommand(cmdList)
 	cmdList.Flags().BoolVar(&listJSON, "json", false, "format output in JSON")
 
 	root.AddCommand(cmdHttpServer)
 	cmdHttpServer.Flags().IntVarP(&httpPort, "port", "P", 8000, "Listen on provided port")
+
+	root.AddCommand(cmdRunUpgrade)
+	cmdRunUpgrade.Flags().BoolVar(&findParentImage, "find-parent-image", false, "automatically find parent image if not provided -- note on qemu, this will download the image")
+	cmdRunUpgrade.Flags().StringVar(&qemuImageDir, "qemu-image-dir", "", "directory in which to cache QEMU images if --fetch-parent-image is enabled")
 }
 
 func main() {
@@ -358,3 +381,179 @@ func runHttpServer(cmd *cobra.Command, args []string) {
 	fmt.Fprintf(os.Stdout, "Serving HTTP on port: %d\n", httpPort)
 	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", httpPort), nil))
 }
+
+func preRunUpgrade(cmd *cobra.Command, args []string) {
+	// unlike `kola run`, we *require* the --cosa-build arg -- XXX: figure out
+	// how to get this working using cobra's MarkFlagRequired()
+	if kola.Options.CosaBuild == "" {
+		fmt.Fprint(os.Stderr, "Error: missing required argument --cosa-build\n")
+		os.Exit(3)
+	}
+
+	err := syncOptions()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(3)
+	}
+
+	if findParentImage {
+		err = syncFindParentImageOptions()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+			os.Exit(3)
+		}
+	}
+}
+
+func postRunUpgrade(cmd *cobra.Command, args []string) {
+	if qemuImageDir != "" && qemuImageDirIsTemp {
+		os.RemoveAll(qemuImageDir)
+	}
+}
+
+// syncFindParentImageOptions handles --find-parent-image automagic.
+func syncFindParentImageOptions() error {
+	var err error
+
+	var parentBaseUrl string
+	switch kola.Options.Distribution {
+	case "fcos":
+		parentBaseUrl, err = getParentFcosBuildBase()
+		if err != nil {
+			return err
+		}
+	default:
+		return fmt.Errorf("--find-parent-image not yet supported for distro %s", kola.Options.Distribution)
+	}
+
+	var parentCosaBuild *cosa.Build
+	parentCosaBuild, err = cosa.FetchAndParseBuild(parentBaseUrl + "meta.json")
+	if err != nil {
+		return err
+	}
+
+	// Here we handle the --fetch-parent-image --> platform-specific options
+	// based on its cosa build metadata
+	switch kolaPlatform {
+	case "qemu-unpriv":
+		if qemuImageDir == "" {
+			if qemuImageDir, err = ioutil.TempDir("", "kola-run-upgrades"); err != nil {
+				return err
+			}
+			qemuImageDirIsTemp = true
+		}
+		qcowUrl := parentBaseUrl + parentCosaBuild.Images.QEMU.Path
+		qcowLocal := filepath.Join(qemuImageDir, parentCosaBuild.Images.QEMU.Path)
+		decompressedQcowLocal, err := downloadImageAndDecompress(qcowUrl, qcowLocal)
+		if err != nil {
+			return err
+		}
+		kola.QEMUOptions.DiskImage = decompressedQcowLocal
+	case "aws":
+		kola.AWSOptions.AMI, err = parentCosaBuild.FindAMI(kola.AWSOptions.Region)
+		if err != nil {
+			return err
+		}
+	default:
+		err = fmt.Errorf("--find-parent-image not yet supported for platform %s", kolaPlatform)
+	}
+
+	return nil
+}
+
+// Note this is a no-op if the decompressed dest already exists.
+func downloadImageAndDecompress(url, compressedDest string) (string, error) {
+	var decompressedDest string
+	if strings.HasSuffix(compressedDest, ".xz") {
+		// if the decompressed file is already present locally, assume it's
+		// good and verified already
+		decompressedDest = strings.TrimSuffix(compressedDest, ".xz")
+		if exists, err := util.PathExists(decompressedDest); err != nil {
+			return "", err
+		} else if exists {
+			return decompressedDest, nil
+		}
+	}
+
+	// XXX enhance API for streaming verification and decompression
+	if err := sdk.DownloadSignedFile(compressedDest, url, nil, ""); err != nil {
+		return "", err
+	}
+
+	if strings.HasSuffix(compressedDest, ".xz") {
+		plog.Infof("Decompressing %s...", compressedDest)
+		if err := util.XZ2File(decompressedDest, compressedDest); err != nil {
+			return "", err
+		}
+		return decompressedDest, nil
+	}
+
+	return compressedDest, nil
+}
+
+func getParentFcosBuildBase() (string, error) {
+	// For FCOS, we can be clever and automagically fetch the metadata for the
+	// parent release, which should be the latest release on that stream.
+
+	// We're taking liberal shortcuts here... the cleaner way to do this is
+	// parse commitmeta.json for `fedora-coreos.stream`, then fetch the stream
+	// metadata for that stream, then fetch the release metadata
+
+	if kola.CosaBuild.Ref == "" {
+		return "", errors.New("no ref in build metadata")
+	}
+
+	stream := filepath.Base(kola.CosaBuild.Ref)
+
+	var parentVersion string
+	if kola.CosaBuild.FedoraCoreOSParentVersion != "" {
+		parentVersion = kola.CosaBuild.FedoraCoreOSParentVersion
+	} else {
+		// ok, we're probably operating on a local dev build since the pipeline
+		// always injects the parent; just instead fetch the release index
+		// for that stream and get the last build id from there
+		index, err := fcos.FetchAndParseCanonicalReleaseIndex(stream)
+		if err != nil {
+			return "", err
+		}
+
+		n := len(index.Releases)
+		if n == 0 {
+			return "", fmt.Errorf("no parent version in build metadata, and no build on stream %s", stream)
+		}
+
+		parentVersion = index.Releases[n-1].Version
+	}
+
+	// XXX: multi-arch
+	// XXX: centralize URL and parameterize
+	return fmt.Sprintf("https://builds.coreos.fedoraproject.org/prod/streams/%s/builds/%s/x86_64/", stream, parentVersion), nil
+}
+
+func runRunUpgrade(cmd *cobra.Command, args []string) {
+	outputDir, err := kola.SetupOutputDir(outputDir, kolaPlatform)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+
+	var patterns []string
+	if len(args) == 0 {
+		patterns = []string{"*"} // run all tests by default
+	} else {
+		patterns = args
+	}
+
+	runErr := kola.RunUpgradeTests(patterns, kolaPlatform, outputDir, !kola.Options.NoTestExitError)
+
+	// needs to be after RunTests() because harness empties the directory
+	if err := writeProps(); err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+
+	if runErr != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", runErr)
+		os.Exit(1)
+	}
+}

cmd/kolet/kolet.go

@@ -82,6 +82,7 @@ func registerTestMap(m map[string]*register.Test) {
 
 func main() {
 	registerTestMap(register.Tests)
+	registerTestMap(register.UpgradeTests)
 	root.AddCommand(cmdRun)
 
 	cli.Execute(root)

kola/README.md

@@ -190,4 +190,4 @@ import (
         _ "github.com/coreos/mantle/kola/tests/systemd"
         _ "github.com/coreos/mantle/kola/tests/update"
 )
-```
+```

kola/harness.go

@@ -431,6 +431,10 @@ func RunTests(patterns []string, pltfrm, outputDir string, propagateTestErrors b
 	return runProvidedTests(register.Tests, patterns, pltfrm, outputDir, propagateTestErrors)
 }
 
+func RunUpgradeTests(patterns []string, pltfrm, outputDir string, propagateTestErrors bool) error {
+	return runProvidedTests(register.UpgradeTests, patterns, pltfrm, outputDir, propagateTestErrors)
+}
+
 // getClusterSemVer returns the CoreOS semantic version via starting a
 // machine and checking
 func getClusterSemver(flight platform.Flight, outputDir string) (*semver.Version, error) {

kola/register/register.go

@@ -82,6 +82,10 @@ type Test struct {
 // to tests.
 var Tests = map[string]*Test{}
 
+// Registered tests that run as part of `kola run-upgrade` live here. Mapping of
+// names to tests.
+var UpgradeTests = map[string]*Test{}
+
 // Register is usually called via init() functions and is how kola test
 // harnesses knows which tests it can choose from. Panics if existing name is
 // registered
@@ -102,6 +106,10 @@ func RegisterTest(t *Test) {
 	Register(Tests, t)
 }
 
+func RegisterUpgradeTest(t *Test) {
+	Register(UpgradeTests, t)
+}
+
 func (t *Test) HasFlag(flag Flag) bool {
 	for _, f := range t.Flags {
 		if f == flag {

kola/registry/registry.go

@@ -16,4 +16,5 @@ import (
 	_ "github.com/coreos/mantle/kola/tests/rhcos"
 	_ "github.com/coreos/mantle/kola/tests/rpmostree"
 	_ "github.com/coreos/mantle/kola/tests/systemd"
+	_ "github.com/coreos/mantle/kola/tests/upgrade"
 )