Add OpenTelemetry tracing to minikube

cmd/minikube/cmd/start.go

@@ -60,6 +60,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/out/register"
 	"k8s.io/minikube/pkg/minikube/reason"
 	"k8s.io/minikube/pkg/minikube/style"
+	pkgtrace "k8s.io/minikube/pkg/trace"
 
 	"k8s.io/minikube/pkg/minikube/registry"
 	"k8s.io/minikube/pkg/minikube/translate"
@@ -129,6 +130,10 @@ func runStart(cmd *cobra.Command, args []string) {
 	register.SetEventLogPath(localpath.EventLog(ClusterFlagValue()))
 
 	out.SetJSON(outputFormat == "json")
+	if err := pkgtrace.Initialize(viper.GetString(trace)); err != nil {
+		exit.Message(reason.Usage, "error initializing tracing: {{.Error}}", out.V{"Error": err.Error()})
+	}
+	defer pkgtrace.Cleanup()
 	displayVersion(version.GetVersion())
 
 	// No need to do the update check if no one is going to see it

cmd/minikube/cmd/start_flags.go

@@ -108,6 +108,7 @@ const (
 	kicBaseImage            = "base-image"
 	ports                   = "ports"
 	startNamespace          = "namespace"
+	trace                   = "trace"
 )
 
 var (
@@ -152,6 +153,7 @@ func initMinikubeFlags() {
 	startCmd.Flags().Bool(deleteOnFailure, false, "If set, delete the current cluster if start fails and try again. Defaults to false.")
 	startCmd.Flags().Bool(forceSystemd, false, "If set, force the container runtime to use sytemd as cgroup manager. Currently available for docker and crio. Defaults to false.")
 	startCmd.Flags().StringVarP(&outputFormat, "output", "o", "text", "Format to print stdout in. Options include: [text,json]")
+	startCmd.Flags().StringP(trace, "", "", "Send trace events. Options include: [gcp]")
 }
 
 // initKubernetesFlags inits the commandline flags for Kubernetes related options

go.mod

@@ -3,8 +3,9 @@ module k8s.io/minikube
 go 1.15
 
 require (
-	cloud.google.com/go/storage v1.8.0
+	cloud.google.com/go/storage v1.10.0
 	github.com/Azure/azure-sdk-for-go v42.3.0+incompatible
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v0.13.0
 	github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 // indirect
 	github.com/Parallels/docker-machine-parallels/v2 v2.0.1
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
@@ -74,21 +75,25 @@ require (
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f
 	github.com/zchee/go-vmnet v0.0.0-20161021174912-97ebf9174097
+	go.opencensus.io v0.22.4
+	go.opentelemetry.io/otel v0.13.0
+	go.opentelemetry.io/otel/sdk v0.13.0
 	golang.org/x/build v0.0.0-20190927031335-2835ba2e683f
 	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a
 	golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6
 	golang.org/x/mod v0.3.0
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
-	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
+	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
 	golang.org/x/sys v0.0.0-20200523222454-059865788121
-	golang.org/x/text v0.3.2
-	google.golang.org/api v0.25.0
+	golang.org/x/text v0.3.3
+	google.golang.org/api v0.29.0
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
 	gopkg.in/yaml.v2 v2.3.0
 	gotest.tools/v3 v3.0.2 // indirect
 	k8s.io/api v0.17.4
 	k8s.io/apimachinery v0.17.4
 	k8s.io/client-go v0.17.4
+	k8s.io/klog v1.0.0
 	k8s.io/klog/v2 v2.4.0
 	k8s.io/kubectl v0.0.0
 	k8s.io/kubernetes v1.18.5

pkg/minikube/out/register/register.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 
 	"k8s.io/klog/v2"
+	"k8s.io/minikube/pkg/trace"
 )
 
 const (
@@ -117,13 +118,16 @@ func (r *Register) currentStep() string {
 
 // SetStep sets the current step
 func (r *Register) SetStep(s RegStep) {
+	defer trace.StartSpan(string(s))
 	if r.first == RegStep("") {
 		_, ok := r.steps[s]
 		if ok {
 			r.first = s
 		} else {
 			klog.Errorf("unexpected first step: %q", r.first)
 		}
+	} else {
+		trace.EndSpan(string(r.current))
 	}
 
 	r.current = s

pkg/trace/gcp.go

@@ -0,0 +1,97 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package trace
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"go.opentelemetry.io/otel/api/trace"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"k8s.io/klog"
+
+	texporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace"
+	"github.com/pkg/errors"
+	"go.opentelemetry.io/otel/api/global"
+)
+
+const (
+	projectEnvVar  = "MINIKUBE_GCP_PROJECT_ID"
+	parentSpanName = "minikube start"
+)
+
+type gcpTracer struct {
+	projectID string
+	parentCtx context.Context
+	trace.Tracer
+	spans   map[string]trace.Span
+	cleanup func()
+}
+
+func (t *gcpTracer) StartSpan(name string) {
+	_, span := t.Tracer.Start(t.parentCtx, name)
+	t.spans[name] = span
+}
+func (t *gcpTracer) EndSpan(name string) {
+	span, ok := t.spans[name]
+	if !ok {
+		klog.Warningf("cannot end span %s as it was never started", name)
+		return
+	}
+	span.End()
+}
+
+func (t *gcpTracer) Cleanup() {
+	span, ok := t.spans[parentSpanName]
+	if ok {
+		span.End()
+	}
+	t.cleanup()
+}
+
+func initGCPTracer() (*gcpTracer, error) {
+	projectID := os.Getenv(projectEnvVar)
+	if projectID == "" {
+		return nil, fmt.Errorf("GCP tracer requires a valid GCP project id set via the %s env variable", projectEnvVar)
+	}
+
+	_, flush, err := texporter.InstallNewPipeline(
+		[]texporter.Option{
+			texporter.WithProjectID(projectID),
+		},
+		sdktrace.WithConfig(sdktrace.Config{
+			DefaultSampler: sdktrace.AlwaysSample(),
+		}),
+	)
+	if err != nil {
+		return nil, errors.Wrap(err, "installing pipeline")
+	}
+
+	t := global.Tracer(parentSpanName)
+
+	ctx, span := t.Start(context.Background(), parentSpanName)
+	return &gcpTracer{
+		projectID: projectID,
+		parentCtx: ctx,
+		cleanup:   flush,
+		Tracer:    t,
+		spans: map[string]trace.Span{
+			parentSpanName: span,
+		},
+	}, nil
+}

pkg/trace/trace.go

@@ -0,0 +1,78 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package trace
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+)
+
+var (
+	tracer minikubeTracer
+)
+
+type minikubeTracer interface {
+	StartSpan(string)
+	EndSpan(string)
+	Cleanup()
+}
+
+// Initialize intializes the global tracer variable
+func Initialize(t string) error {
+	tr, err := getTracer(t)
+	if err != nil {
+		return errors.Wrap(err, "getting tracer")
+	}
+	tracer = tr
+	return nil
+}
+
+func getTracer(t string) (minikubeTracer, error) {
+	switch t {
+	case "gcp":
+		return initGCPTracer()
+	case "":
+		return nil, nil
+	}
+	return nil, fmt.Errorf("%s is not a valid tracer", t)
+}
+
+// StartSpan starts a span with the given name
+func StartSpan(name string) {
+	if tracer == nil {
+		return
+	}
+	tracer.StartSpan(name)
+}
+
+// EndSpan ends a span with the given name
+func EndSpan(name string) {
+	if tracer == nil {
+		return
+	}
+	tracer.EndSpan(name)
+}
+
+// Cleanup is responsible for trace related cleanup,
+// such as flushing all data
+func Cleanup() {
+	if tracer == nil {
+		return
+	}
+	tracer.Cleanup()
+}