test: implement talosctl conformance command to run e2e tests

Command implements two modes:

* `fast`: conformance suite is run at maximum speed
* `certified`: conformance suite is run in serial mode, results
  are capture to produce artifacts ready for CNCF submission process

Signed-off-by: Andrey Smirnov <smirnov.andrey@gmail.com>

.drone.jsonnet

@@ -468,9 +468,12 @@ local e2e_pipelines = [
 
 // Conformance pipeline.
 
-local conformance_aws = Step("e2e-aws", depends_on=[e2e_capi], environment=creds_env_vars+{SONOBUOY_MODE: "certified-conformance"});
-local conformance_azure = Step("e2e-azure", depends_on=[e2e_capi], environment=creds_env_vars+{SONOBUOY_MODE: "certified-conformance"});
-local conformance_gcp = Step("e2e-gcp", depends_on=[e2e_capi], environment=creds_env_vars+{SONOBUOY_MODE: "certified-conformance"});
+local conformance_k8s_qemu = Step("conformance-k8s-qemu", target="e2e-qemu", privileged=true, depends_on=[load_artifacts], environment={
+        "QEMU_WORKERS": "2", // conformance test requires >=2 workers
+        "QEMU_CPUS": "4", // conformance test in parallel runs with number of CPUs
+        "TEST_MODE": "fast-conformance",
+        "IMAGE_REGISTRY": local_registry,
+});
 
 local conformance_trigger(names) = {
   trigger: {
@@ -481,8 +484,11 @@ local conformance_trigger(names) = {
 };
 
 local conformance_pipelines = [
-  Pipeline('conformance-aws', default_pipeline_steps + [capi_docker, e2e_capi, conformance_aws]) + conformance_trigger(['conformance-aws']),
-  Pipeline('conformance-gcp', default_pipeline_steps + [capi_docker, e2e_capi, conformance_gcp]) + conformance_trigger(['conformance-gcp']),
+  // regular pipelines, triggered on promote events
+  Pipeline('conformance-qemu', default_pipeline_steps + [conformance_k8s_qemu]) + conformance_trigger(['conformance-qemu']),
+
+  // cron pipelines, triggered on schedule events
+  Pipeline('cron-conformance-qemu', default_pipeline_steps + [conformance_k8s_qemu]) + cron_trigger(['nightly']),
 ];
 
 // Cloud images pipeline.

cmd/talosctl/cmd/talos/conformance.go

@@ -0,0 +1,67 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package talos
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"github.com/talos-systems/talos/pkg/cluster"
+	"github.com/talos-systems/talos/pkg/cluster/sonobuoy"
+	"github.com/talos-systems/talos/pkg/machinery/client"
+)
+
+// conformanceCmd represents the conformance command.
+var conformanceCmd = &cobra.Command{
+	Use:   "conformance",
+	Short: "Run conformance tests",
+	Long:  ``,
+}
+
+var conformanceKubernetesCmdFlags struct {
+	mode string
+}
+
+var conformanceKubernetesCmd = &cobra.Command{
+	Use:     "kubernetes",
+	Aliases: []string{"k8s"},
+	Short:   "Run Kubernetes conformance tests",
+	Long:    ``,
+	Args:    cobra.NoArgs,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return WithClient(func(ctx context.Context, c *client.Client) error {
+			clientProvider := &cluster.ConfigClientProvider{
+				DefaultClient: c,
+			}
+			defer clientProvider.Close() //nolint:errcheck
+
+			state := struct {
+				cluster.K8sProvider
+			}{
+				K8sProvider: &cluster.KubernetesClient{
+					ClientProvider: clientProvider,
+					ForceEndpoint:  healthCmdFlags.forceEndpoint,
+				},
+			}
+
+			switch conformanceKubernetesCmdFlags.mode {
+			case "fast":
+				return sonobuoy.FastConformance(ctx, &state)
+			case "certified":
+				return sonobuoy.CertifiedConformance(ctx, &state)
+			default:
+				return fmt.Errorf("unsupported conformance mode %v", conformanceKubernetesCmdFlags.mode)
+			}
+		})
+	},
+}
+
+func init() {
+	conformanceKubernetesCmd.Flags().StringVar(&conformanceKubernetesCmdFlags.mode, "mode", "fast", "conformance test mode: [fast, certified]")
+	conformanceCmd.AddCommand(conformanceKubernetesCmd)
+	addCommand(conformanceCmd)
+}

hack/test/e2e-qemu.sh

@@ -67,9 +67,10 @@ function create_cluster {
     --provisioner "${PROVISIONER}" \
     --name "${CLUSTER_NAME}" \
     --masters=3 \
+    --workers="${QEMU_WORKERS:-1}" \
     --mtu 1450 \
     --memory 2048 \
-    --cpus 2.0 \
+    --cpus "${QEMU_CPUS:-2}" \
     --cidr 172.20.1.0/24 \
     --user-disk /var/lib/extra:100MB \
     --user-disk /var/lib/p1:100MB:/var/lib/p2:100MB \
@@ -91,7 +92,17 @@ function destroy_cluster() {
 }
 
 create_cluster
-get_kubeconfig
-run_talos_integration_test
-run_kubernetes_integration_test
+
+case "${TEST_MODE:-default}" in
+  fast-conformance)
+    run_kubernetes_conformance_test fast
+    ;;
+  *)
+    get_kubeconfig
+    run_talos_integration_test
+    run_kubernetes_integration_test
+    ;;
+esac
+
+
 destroy_cluster

hack/test/e2e.sh

@@ -169,6 +169,10 @@ function run_talos_integration_test_docker {
   "${INTEGRATION_TEST}" -test.v -talos.talosctlpath "${TALOSCTL}" -talos.kubectlpath "${KUBECTL}" -talos.k8sendpoint 127.0.0.1:6443 -talos.provisioner "${PROVISIONER}" -talos.name "${CLUSTER_NAME}" ${TEST_RUN} ${TEST_SHORT}
 }
 
+function run_kubernetes_conformance_test {
+  "${TALOSCTL}" conformance kubernetes --mode="${1}"
+}
+
 function run_kubernetes_integration_test {
   timeout=$(($(date +%s) + ${TIMEOUT}))
   until ${SONOBUOY} run \

pkg/cluster/sonobuoy/product.go

@@ -0,0 +1,31 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package sonobuoy
+
+import "github.com/talos-systems/talos/pkg/version"
+
+type product struct {
+	Vendor           string `yaml:"vendor"`
+	Name             string `yaml:"name"`
+	Version          string `yaml:"version"`
+	WebsiteURL       string `yaml:"website_url"`
+	RepoURL          string `yaml:"repo_url"`
+	DocumentationURL string `yaml:"documentation_url"`
+	ProductLogoURL   string `yaml:"product_logo_url"`
+	Type             string `yaml:"type"`
+	Description      string `yaml:"description"`
+}
+
+var talos = product{
+	Vendor:           "Talos Systems",
+	Name:             "Talos",
+	Version:          version.Tag,
+	WebsiteURL:       "https://www.talos-systems.com",
+	RepoURL:          "https://github.com/talos-systems/talos",
+	DocumentationURL: "https://www.talos.dev",
+	ProductLogoURL:   "https://www.talos-systems.com/images/TalosSystems_Horizontal_Logo_FullColor_RGB-for-site.svg",
+	Type:             "installer",
+	Description:      "Talos is a modern Kubernetes-focused OS designed to be secure, immutable, and minimal.",
+}

pkg/cluster/sonobuoy/sonobuoy.go

@@ -6,18 +6,23 @@
 package sonobuoy
 
 import (
+	"archive/tar"
+	"compress/gzip"
 	"context"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
+	"github.com/coreos/go-semver/semver"
 	"github.com/vmware-tanzu/sonobuoy/cmd/sonobuoy/app"
 	"github.com/vmware-tanzu/sonobuoy/pkg/client"
 	"github.com/vmware-tanzu/sonobuoy/pkg/config"
 	sonodynamic "github.com/vmware-tanzu/sonobuoy/pkg/dynamic"
+	"gopkg.in/yaml.v3"
 
 	"github.com/talos-systems/talos/pkg/cluster"
 	"github.com/talos-systems/talos/pkg/machinery/constants"
@@ -26,13 +31,17 @@ import (
 // Options for the tests.
 type Options struct {
 	RunTests []string
+	Parallel bool
 
 	RunTimeout    time.Duration
 	DeleteTimeout time.Duration
 
 	KubernetesVersion string
 
-	UseSpinner bool
+	UseSpinner      bool
+	RetrieveResults bool
+
+	ResultsPath string
 }
 
 // DefaultOptions with hand-picked tests, timeouts, etc.
@@ -51,9 +60,70 @@ func DefaultOptions() *Options {
 	}
 }
 
+// FastConformance runs conformance suite in two passes: parallel + serial for non parallel-safe tests.
+func FastConformance(ctx context.Context, cluster cluster.K8sProvider) error {
+	optionsList := []Options{
+		{
+			RunTests: []string{`\[Conformance\]`},
+			Parallel: true,
+
+			RunTimeout:    time.Hour,
+			DeleteTimeout: 5 * time.Minute,
+
+			KubernetesVersion: constants.DefaultKubernetesVersion,
+		},
+		{
+			RunTests: []string{`\[Serial\].*\[Conformance\]`},
+			Parallel: false,
+
+			RunTimeout:    time.Hour,
+			DeleteTimeout: 5 * time.Minute,
+
+			KubernetesVersion: constants.DefaultKubernetesVersion,
+		},
+	}
+
+	for _, options := range optionsList {
+		options := options
+
+		if err := Run(ctx, cluster, &options); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// CertifiedConformance runs conformance suite in certified mode collecting all the results.
+func CertifiedConformance(ctx context.Context, cluster cluster.K8sProvider) error {
+	options := Options{
+		RunTests: []string{`\[Conformance\]`},
+		Parallel: false,
+
+		RunTimeout:    2 * time.Hour,
+		DeleteTimeout: 5 * time.Minute,
+
+		KubernetesVersion: constants.DefaultKubernetesVersion,
+		RetrieveResults:   true,
+	}
+
+	k8sVersion, err := semver.NewVersion(options.KubernetesVersion)
+	if err != nil {
+		return err
+	}
+
+	options.ResultsPath = fmt.Sprintf("v%d.%d/talos", k8sVersion.Major, k8sVersion.Minor)
+
+	if err = os.MkdirAll(options.ResultsPath, 0o755); err != nil {
+		return err
+	}
+
+	return Run(ctx, cluster, &options)
+}
+
 // Run the e2e test against cluster with provided options.
 //
-//nolint:gocyclo
+//nolint:gocyclo,cyclop
 func Run(ctx context.Context, cluster cluster.K8sProvider, options *Options) error {
 	var waitOutput string
 
@@ -119,7 +189,7 @@ func Run(ctx context.Context, cluster cluster.K8sProvider, options *Options) err
 
 	runConfig.E2EConfig = &client.E2EConfig{
 		Focus:    strings.Join(options.RunTests, "|"),
-		Parallel: "false",
+		Parallel: fmt.Sprintf("%v", options.Parallel),
 	}
 	runConfig.DynamicPlugins = []string{"e2e"}
 	runConfig.KubeConformanceImage = fmt.Sprintf("%s:v%s", config.UpstreamKubeConformanceImageURL, options.KubernetesVersion)
@@ -164,5 +234,98 @@ func Run(ctx context.Context, cluster cluster.K8sProvider, options *Options) err
 		return fmt.Errorf("missing e2e plugin status")
 	}
 
+	if options.RetrieveResults {
+		resultR, errCh, err := sclient.RetrieveResults(&client.RetrieveConfig{
+			Namespace: config.DefaultNamespace,
+		})
+		if err != nil {
+			return fmt.Errorf("error retrieving results: %w", err)
+		}
+
+		if resultR == nil {
+			return fmt.Errorf("no result reader")
+		}
+
+		tarR := tar.NewReader(resultR)
+
+		for {
+			var header *tar.Header
+
+			header, err = tarR.Next()
+			if err != nil {
+				if err == io.EOF {
+					break
+				}
+
+				return err
+			}
+
+			matched, _ := filepath.Match("tmp/sonobuoy/*_sonobuoy_*.tar.gz", header.Name) //nolint: errcheck
+
+			if !matched {
+				continue
+			}
+
+			var gzipR *gzip.Reader
+
+			gzipR, err = gzip.NewReader(tarR)
+			if err != nil {
+				return err
+			}
+
+			defer gzipR.Close() //nolint: errcheck
+
+			innnerTarR := tar.NewReader(gzipR)
+
+			for {
+				header, err = innnerTarR.Next()
+				if err != nil {
+					if err == io.EOF {
+						break
+					}
+
+					return err
+				}
+
+				writeFile := func(name string) error {
+					var data []byte
+
+					data, err = io.ReadAll(innnerTarR)
+					if err != nil {
+						return err
+					}
+
+					return os.WriteFile(filepath.Join(options.ResultsPath, name), data, 0o644)
+				}
+
+				switch header.Name {
+				case "plugins/e2e/results/global/junit_01.xml":
+					if err = writeFile("junit_01.xml"); err != nil {
+						return err
+					}
+				case "plugins/e2e/results/global/e2e.log":
+					if err = writeFile("e2e.log"); err != nil {
+						return err
+					}
+				}
+			}
+		}
+
+		select {
+		case err = <-errCh:
+			return err
+		default:
+		}
+
+		productInfo, err := yaml.Marshal(talos)
+		if err != nil {
+			return fmt.Errorf("error marshaling product info: %w", err)
+		}
+
+		if err = os.WriteFile(filepath.Join(options.ResultsPath, "PRODUCT.yaml"), productInfo, 0o644); err != nil {
+			return err
+		}
+	}
+
 	return cleanup()
 }