Merge pull request #16220 from rifelpet/jaeger

Jaeger tracing visualizer improvements

docs/opentelemetry.md

@@ -12,12 +12,47 @@ To try this out:
 
 You should now see that the /tmp/trace file is created.
 
-Then we have an experimental tool to serve the trace file to jaeger:
+Then we have an experimental tool to serve the trace file to jaeger. First ensure `jaeger-query` is in your PATH, available [here](https://www.jaegertracing.io/download/#binaries)
 
 ```
 cd tools/otel/traceserver
 go run . --src /tmp/trace --run jaeger
 ```
 
+Alternatively you can use the jaeger docker image with:
+
+```
+go run . --src /tmp/trace --run docker-jaeger
+```
+
+This will open your local jaeger-query's frontend in a web browser:
+
+![Jaeger UI](./img/jaeger-query.png)
+
 Not everything is instrumented yet, and not all the traces are fully joined up (we need to thread more contexts through more methods),
 but you should be able to start to explore the operations that we run and their performance.
+
+## Prow Jobs
+
+Tracing is enabled in the kops prow jobs. `.otel` files are created during job execution and included as job artifacts in the `otlp` subdirectory.
+
+Example:
+
+```
+https://gcsweb.k8s.io/gcs/kubernetes-jenkins/pr-logs/pull/kops/16204/presubmit-kops-aws-scale-amazonvpc-using-cl2/1742962895290896384/artifacts/otlp/
+```
+
+Download these files to a local directory, either with `gsutil` or through a web browser
+
+```
+mkdir /tmp/job-traces
+
+gsutil cp -r gs://kubernetes-jenkins/pr-logs/pull/kops/16204/presubmit-kops-aws-scale-amazonvpc-using-cl2/1742962895290896384/artifacts/otlp/ /tmp/job-traces
+```
+
+Then run the trace server as normal:
+
+```sh
+cd tools/otel/traceserver
+go run . --src /tmp/traces --run jaeger
+```

tools/otel/traceserver/main.go

@@ -58,7 +58,7 @@ func run(ctx context.Context) error {
 	src := ""
 	flag.StringVar(&src, "src", src, "tracefile to load")
 	flag.StringVar(&listen, "listen", listen, "endpoint on which to serve grpc")
-	flag.StringVar(&run, "run", run, "visualization program to run [jaeger]")
+	flag.StringVar(&run, "run", run, "visualization program to run [jaeger, docker-jaeger]")
 	klog.InitFlags(nil)
 	flag.Parse()
 
@@ -68,18 +68,19 @@ func run(ctx context.Context) error {
 
 	if run != "" {
 		switch run {
-		case "jaeger":
+		case "jaeger", "docker-jaeger":
 			go func() {
 				opt := RunJaegerOptions{
 					StorageServer: listen,
+					UseDocker:     run == "docker-jaeger",
 				}
 				err := runJaeger(ctx, opt)
 				if err != nil {
 					klog.Warningf("error starting jaeger: %w", err)
 				}
 			}()
 		default:
-			return fmt.Errorf("run=%q not known (valid values: jaeger)", run)
+			return fmt.Errorf("run=%q not known (valid values: jaeger, docker-jaeger)", run)
 		}
 	}
 
@@ -485,6 +486,7 @@ func attributeValueAsString(v *v11.AnyValue) (string, error) {
 // RunJaegerOptions are the options for runJaeger
 type RunJaegerOptions struct {
 	StorageServer string
+	UseDocker     bool
 }
 
 // runJaeger starts the jaeger query & visualizer, binding to our storage server
@@ -494,25 +496,45 @@ func runJaeger(ctx context.Context, opt RunJaegerOptions) error {
 	var jaeger *exec.Cmd
 	{
 		klog.Infof("starting jaeger")
-		args := []string{
-			"docker", "run", "--rm", "--network=host", "--name=jaeger",
-			"-e=SPAN_STORAGE_TYPE=grpc-plugin",
-			"jaegertracing/jaeger-query",
-			"--grpc-storage.server=" + opt.StorageServer,
+
+		var c *exec.Cmd
+		if opt.UseDocker {
+			args := []string{
+				"docker", "run", "--rm", "--network=host", "--name=jaeger",
+				"-e=SPAN_STORAGE_TYPE=grpc-plugin",
+				"jaegertracing/jaeger-query",
+				"--grpc-storage.server=" + opt.StorageServer,
+			}
+			c = exec.CommandContext(ctx, args[0], args[1:]...)
+		} else {
+			args := []string{
+				"jaeger-query",
+				"--grpc-storage.server=" + opt.StorageServer,
+			}
+			c = exec.CommandContext(ctx, args[0], args[1:]...)
+			c.Env = append(os.Environ(), "SPAN_STORAGE_TYPE=grpc-plugin")
 		}
 
-		c := exec.CommandContext(ctx, args[0], args[1:]...)
 		c.Stdout = os.Stdout
 		c.Stderr = os.Stderr
 		if err := c.Start(); err != nil {
-			return fmt.Errorf("starting jaeger in docker (%s): %w", strings.Join(args, " "), err)
+			return fmt.Errorf("starting jaeger (%s): %w", strings.Join(c.Args, " "), err)
 		}
 		jaeger = c
 	}
 
 	{
 		fmt.Fprintf(os.Stdout, "open browser to %s\n", jaegerURL)
-		args := []string{"xdg-open", jaegerURL}
+
+		args := make([]string, 0)
+		for _, o := range []string{"xdg-open", "open"} {
+			if _, err := exec.LookPath(o); err == nil {
+				args = append(args, o)
+				break
+			}
+		}
+		args = append(args, jaegerURL)
+
 		c := exec.CommandContext(ctx, args[0], args[1:]...)
 		c.Stdout = os.Stdout
 		c.Stderr = os.Stderr