diff --git a/cmd/skaffold/app/cmd/runner.go b/cmd/skaffold/app/cmd/runner.go index 99802665f6a..fa607b350ec 100644 --- a/cmd/skaffold/app/cmd/runner.go +++ b/cmd/skaffold/app/cmd/runner.go @@ -64,13 +64,16 @@ func createNewRunner(out io.Writer, opts config.SkaffoldOptions) (runner.Runner, return nil, nil, nil, err } - instrumentation.InitMeterFromConfig(configs, opts.User) runner, err := v1.NewForConfig(runCtx) if err != nil { event.InititializationFailed(err) return nil, nil, nil, fmt.Errorf("creating runner: %w", err) } - + instrumentation.InitMeterFromConfig(configs, opts.User) + _, _, err = instrumentation.InitTraceFromEnvVar() + if err != nil { + logrus.Debugf("error initializing tracing: %v", err) + } return runner, configs, runCtx, nil } diff --git a/cmd/skaffold/skaffold.go b/cmd/skaffold/skaffold.go index d0659d3eedf..f7f37b392a6 100644 --- a/cmd/skaffold/skaffold.go +++ b/cmd/skaffold/skaffold.go @@ -49,6 +49,10 @@ func main() { if err := instrumentation.ExportMetrics(code); err != nil { logrus.Debugf("error exporting metrics %v", err) } + if err := instrumentation.TracerShutdown(context.Background()); err != nil { + logrus.Debugf("error shutting down tracer %v", err) + } + os.Exit(code) } diff --git a/examples/jaeger-skaffold-trace/README.md b/examples/jaeger-skaffold-trace/README.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/examples/jaeger-skaffold-trace/jaeger-all-in-one-template.yaml b/examples/jaeger-skaffold-trace/jaeger-all-in-one-template.yaml new file mode 100644 index 00000000000..50023e0faee --- /dev/null +++ b/examples/jaeger-skaffold-trace/jaeger-all-in-one-template.yaml @@ -0,0 +1,161 @@ +# Copyright 2019 The Skaffold Authors + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: List +items: +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: jaeger + labels: + app: jaeger + app.kubernetes.io/name: jaeger + app.kubernetes.io/component: all-in-one + spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: jaeger + app.kubernetes.io/name: jaeger + app.kubernetes.io/component: all-in-one + template: + metadata: + labels: + app: jaeger + app.kubernetes.io/name: jaeger + app.kubernetes.io/component: all-in-one + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "16686" + spec: + containers: + - env: + - name: COLLECTOR_ZIPKIN_HTTP_PORT + value: "9411" + image: jaegertracing/all-in-one + name: jaeger + ports: + - containerPort: 5775 + protocol: UDP + - containerPort: 6831 + protocol: UDP + - containerPort: 6832 + protocol: UDP + - containerPort: 5778 + protocol: TCP + - containerPort: 16686 + protocol: TCP + - containerPort: 9411 + protocol: TCP + readinessProbe: + httpGet: + path: "/" + port: 14269 + initialDelaySeconds: 5 +- apiVersion: v1 + kind: Service + metadata: + name: jaeger-query + labels: + app: jaeger + app.kubernetes.io/name: jaeger + app.kubernetes.io/component: query + spec: + ports: + - name: query-http + port: 16686 + protocol: TCP + targetPort: 16686 + selector: + app.kubernetes.io/name: jaeger + app.kubernetes.io/component: all-in-one + type: LoadBalancer +- apiVersion: v1 + kind: Service + metadata: + name: jaeger-collector + labels: + app: jaeger + app.kubernetes.io/name: jaeger + app.kubernetes.io/component: collector + spec: + ports: + - name: jaeger-collector-tchannel + port: 14267 + protocol: TCP + targetPort: 14267 + - name: jaeger-collector-http + port: 14268 + protocol: TCP + targetPort: 14268 + - name: jaeger-collector-zipkin + port: 9411 + protocol: TCP + targetPort: 9411 + selector: + app.kubernetes.io/name: jaeger + app.kubernetes.io/component: all-in-one + type: ClusterIP +- apiVersion: v1 + kind: Service + metadata: + name: jaeger-agent + labels: + app: jaeger + app.kubernetes.io/name: jaeger + app.kubernetes.io/component: agent + spec: + ports: + - name: agent-zipkin-thrift + port: 5775 + protocol: UDP + targetPort: 5775 + - name: agent-compact + port: 6831 + protocol: UDP + targetPort: 6831 + - name: agent-binary + port: 6832 + protocol: UDP + targetPort: 6832 + - name: agent-configs + port: 5778 + protocol: TCP + targetPort: 5778 + clusterIP: None + selector: + app.kubernetes.io/name: jaeger + app.kubernetes.io/component: all-in-one +- apiVersion: v1 + kind: Service + metadata: + name: zipkin + labels: + app: jaeger + app.kubernetes.io/name: jaeger + app.kubernetes.io/component: zipkin + spec: + ports: + - name: jaeger-collector-zipkin + port: 9411 + protocol: TCP + targetPort: 9411 + clusterIP: None + selector: + app.kubernetes.io/name: jaeger + app.kubernetes.io/component: all-in-one + diff --git a/examples/jaeger-skaffold-trace/skaffold.yaml b/examples/jaeger-skaffold-trace/skaffold.yaml new file mode 100644 index 00000000000..2fd08c1e6ff --- /dev/null +++ b/examples/jaeger-skaffold-trace/skaffold.yaml @@ -0,0 +1,8 @@ +apiVersion: skaffold/v2beta15 +kind: Config +metadata: + name: jaeger-all-in-one +deploy: + kubectl: + manifests: + - jaeger-all-in-one-template.yaml diff --git a/go.mod b/go.mod index 7493b3fcfe6..88884f4a244 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( cloud.google.com/go/storage v1.10.0 github.com/AlecAivazis/survey/v2 v2.2.7 github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.20.0 + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v0.20.0 github.com/aws/aws-sdk-go v1.36.30 // indirect github.com/blang/semver v3.5.1+incompatible github.com/bmatcuk/doublestar v1.2.4 @@ -24,7 +25,6 @@ require ( github.com/buildpacks/lifecycle v0.10.2 github.com/buildpacks/pack v0.17.0 github.com/cenkalti/backoff/v4 v4.0.2 - github.com/daixiang0/gci v0.2.8 // indirect github.com/docker/cli v20.10.0-beta1.0.20201117192004-5cc239616494+incompatible github.com/docker/distribution v2.7.1+incompatible github.com/docker/docker v20.10.0-beta1.0.20201110211921-af34b94a78a1+incompatible @@ -67,8 +67,10 @@ require ( github.com/spf13/pflag v1.0.5 github.com/tektoncd/pipeline v0.5.1-0.20190731183258-9d7e37e85bf8 github.com/xeipuuv/gojsonschema v1.2.0 + go.opencensus.io v0.22.5 go.opentelemetry.io/otel v0.20.0 go.opentelemetry.io/otel/exporters/stdout v0.20.0 + go.opentelemetry.io/otel/exporters/trace/jaeger v0.20.0 // indirect go.opentelemetry.io/otel/metric v0.20.0 go.opentelemetry.io/otel/sdk v0.20.0 go.opentelemetry.io/otel/sdk/metric v0.20.0 diff --git a/go.sum b/go.sum index cb72dec0ee1..176348afedb 100644 --- a/go.sum +++ b/go.sum @@ -23,6 +23,7 @@ cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6 cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.61.0/go.mod h1:XukKJg4Y7QsUu0Hxg3qQKUWR4VuWivmyMK2+rUyxAqw= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0 h1:eWRCuwubtDrCJG0oSUMgnsbD4CmPFQF2ei4OFbXvwww= @@ -132,6 +133,8 @@ github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190822182118-27a4ced3 github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20200415212048-7901bc822317/go.mod h1:DF8FZRxMHMGv/vP2lQP6h+dYzzjpuRn24VeRiYn3qjQ= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.20.0 h1:H4Hs0jLf0IcQ96dgx3gPFRke2zyjrzpYMecVsrWKrAc= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.20.0/go.mod h1:3DfYfW/GJ2p+Yd1vGImcFO1jKaqvZmAMMIdWk8BJsjw= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v0.20.0 h1:ExGRyJwOUijAPv/RzCJ3p1CNUxBQGzVO238m1lFjLS4= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v0.20.0/go.mod h1:f4BFp2+kV6s/OKj3IP/34keB/OE7tTTaZZQyX/mQ7Ng= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd h1:sjQovDkwrZp8u+gxLtPgKGjk5hCxuy2hrRejBTA9xFU= github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= @@ -349,8 +352,6 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= -github.com/daixiang0/gci v0.2.8 h1:1mrIGMBQsBu0P7j7m1M8Lb+ZeZxsZL+jyGX4YoMJJpg= -github.com/daixiang0/gci v0.2.8/go.mod h1:+4dZ7TISfSmqfAGv59ePaHfNzgGtIkHAhhdKggP1JAc= github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -429,6 +430,8 @@ github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZM github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= +github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= @@ -1335,10 +1338,16 @@ go.opencensus.io v0.22.4-0.20200608061201-1901b56b9515/go.mod h1:yxeiOL68Rb0Xd1d go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opentelemetry.io/contrib v0.20.0 h1:ubFQUn0VCZ0gPwIoJfBJVpeBlyRMxu8Mm/huKWYd9p0= +go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0 h1:Q3C9yzW6I9jqEc8sawxzxZmY48fs9u220KXq6d5s3XU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= go.opentelemetry.io/otel v0.20.0 h1:eaP0Fqu7SXHwvjiqDq83zImeehOHX8doTvU9AwXON8g= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= go.opentelemetry.io/otel/exporters/stdout v0.20.0 h1:NXKkOWV7Np9myYrQE0wqRS3SbwzbupHu07rDONKubMo= go.opentelemetry.io/otel/exporters/stdout v0.20.0/go.mod h1:t9LUU3JvYlmoPA61abhvsXxKh58xdyi3nMtI6JiR8v0= +go.opentelemetry.io/otel/exporters/trace/jaeger v0.20.0 h1:FoclOadJNul1vUiKnZU0sKFWOZtZQq3jUzSbrX2jwNM= +go.opentelemetry.io/otel/exporters/trace/jaeger v0.20.0/go.mod h1:10qwvAmKpvwRO5lL3KQ8EWznPp89uGfhcbK152LFWsQ= go.opentelemetry.io/otel/metric v0.20.0 h1:4kzhXFP+btKm4jwxpjIqjs41A7MakRFUS86bqLHTIw8= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= go.opentelemetry.io/otel/oteltest v0.20.0 h1:HiITxCawalo5vQzdHfKeZurV8x7ljcqAgiWzF6Vaeaw= @@ -1713,13 +1722,13 @@ golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200701151220-7cb253f4c4f8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200713011307-fd294ab11aed/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20200916195026-c9a70fc28ce3/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201118003311-bd56c0adb394/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= @@ -1816,6 +1825,8 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200605102947-12044bf5ea91/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200711021454-869866162049/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200715011427-11fb19a81f2c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= diff --git a/pkg/skaffold/build/cache/hash.go b/pkg/skaffold/build/cache/hash.go index fcb7910f369..90f37fd24b9 100644 --- a/pkg/skaffold/build/cache/hash.go +++ b/pkg/skaffold/build/cache/hash.go @@ -33,6 +33,7 @@ import ( "github.com/GoogleContainerTools/skaffold/pkg/skaffold/config" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/graph" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/instrumentation" latestV1 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v1" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/util" ) @@ -66,6 +67,9 @@ func newArtifactHasher(artifacts graph.ArtifactGraph, lister DependencyLister, m } func (h *artifactHasherImpl) hash(ctx context.Context, a *latestV1.Artifact) (string, error) { + ctx, endTrace := instrumentation.StartTrace(ctx, "pkg/skaffold/build/cache", "GenerateHashOneArtifact") + defer endTrace() + hash, err := h.safeHash(ctx, a) if err != nil { return "", err @@ -82,6 +86,7 @@ func (h *artifactHasherImpl) hash(ctx context.Context, a *latestV1.Artifact) (st if len(hashes) == 1 { return hashes[0], nil } + return encode(hashes) } diff --git a/pkg/skaffold/build/cache/lookup.go b/pkg/skaffold/build/cache/lookup.go index 3650f83211d..d5e4d236308 100644 --- a/pkg/skaffold/build/cache/lookup.go +++ b/pkg/skaffold/build/cache/lookup.go @@ -25,6 +25,7 @@ import ( "github.com/sirupsen/logrus" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/instrumentation" latestV1 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v1" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/tag" ) @@ -33,6 +34,9 @@ func (c *cache) lookupArtifacts(ctx context.Context, tags tag.ImageTags, artifac details := make([]cacheDetails, len(artifacts)) // Create a new `artifactHasher` on every new dev loop. // This way every artifact hash is calculated at most once in a single dev loop, and recalculated on every dev loop. + + ctx, endTrace := instrumentation.StartTrace(ctx, "pkg/skaffold/build/cache", "CacheLookupArtifacts") + defer endTrace() h := newArtifactHasherFunc(c.artifactGraph, c.lister, c.cfg.Mode()) var wg sync.WaitGroup for i := range artifacts { @@ -50,6 +54,9 @@ func (c *cache) lookupArtifacts(ctx context.Context, tags tag.ImageTags, artifac } func (c *cache) lookup(ctx context.Context, a *latestV1.Artifact, tag string, h artifactHasher) cacheDetails { + ctx, endTrace := instrumentation.StartTrace(ctx, "pkg/skaffold/build/cache", "CacheLookupOneArtifact") + defer endTrace() + hash, err := h.hash(ctx, a) if err != nil { return failed{err: fmt.Errorf("getting hash for artifact %q: %s", a.ImageName, err)} diff --git a/pkg/skaffold/build/cache/retrieve.go b/pkg/skaffold/build/cache/retrieve.go index dacb013574b..9ae2a7dec7f 100644 --- a/pkg/skaffold/build/cache/retrieve.go +++ b/pkg/skaffold/build/cache/retrieve.go @@ -29,6 +29,7 @@ import ( "github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker" sErrors "github.com/GoogleContainerTools/skaffold/pkg/skaffold/errors" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/graph" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/instrumentation" latestV1 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v1" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/tag" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/util" @@ -40,6 +41,9 @@ func (c *cache) Build(ctx context.Context, out io.Writer, tags tag.ImageTags, ar start := time.Now() + ctx, endTrace := instrumentation.StartTrace(ctx, "pkg/skaffold/runner", "CheckCache") + defer endTrace() + color.Default.Fprintln(out, "Checking cache...") lookup := make(chan []cacheDetails) diff --git a/pkg/skaffold/build/scheduler.go b/pkg/skaffold/build/scheduler.go index 955fbb9ccdb..44a0a1e8fa0 100644 --- a/pkg/skaffold/build/scheduler.go +++ b/pkg/skaffold/build/scheduler.go @@ -27,6 +27,7 @@ import ( "github.com/GoogleContainerTools/skaffold/pkg/skaffold/event" eventV2 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/event/v2" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/graph" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/instrumentation" latestV1 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v1" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/tag" ) @@ -90,6 +91,9 @@ func (s *scheduler) build(ctx context.Context, tags tag.ImageTags, i int) error event.BuildInProgress(a.ImageName) eventV2.BuildInProgress(i, a.ImageName) + ctx, endTrace := instrumentation.StartTrace(ctx, "pkg/skaffold/build", "BuildInProgress", + instrumentation.TraceAttribute{Key: "ImageName", Value: a.ImageName}) + defer endTrace() w, closeFn, err := s.logger.GetWriter() if err != nil { diff --git a/pkg/skaffold/deploy/kpt/kpt.go b/pkg/skaffold/deploy/kpt/kpt.go index 71d86847a22..eea6488d931 100644 --- a/pkg/skaffold/deploy/kpt/kpt.go +++ b/pkg/skaffold/deploy/kpt/kpt.go @@ -383,7 +383,7 @@ func (k *Deployer) renderManifests(ctx context.Context, _ io.Writer, builds []gr if err != nil { return nil, fmt.Errorf("excluding kpt functions from manifests: %w", err) } - manifests, err = manifests.ReplaceImages(builds) + manifests, err = manifests.ReplaceImages(ctx, builds) if err != nil { return nil, fmt.Errorf("replacing images in manifests: %w", err) } diff --git a/pkg/skaffold/deploy/kubectl/kubectl.go b/pkg/skaffold/deploy/kubectl/kubectl.go index 8919a38165d..966219c6ad8 100644 --- a/pkg/skaffold/deploy/kubectl/kubectl.go +++ b/pkg/skaffold/deploy/kubectl/kubectl.go @@ -34,6 +34,7 @@ import ( deployutil "github.com/GoogleContainerTools/skaffold/pkg/skaffold/deploy/util" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/event" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/graph" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/instrumentation" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes/manifest" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/log" @@ -100,15 +101,21 @@ func (k *Deployer) Deploy(ctx context.Context, out io.Writer, builds []graph.Art // also, manually set the labels to ensure the runID is added switch { case len(k.hydratedManifests) > 0: + _, endTrace := instrumentation.StartTrace(ctx, "pkg/skaffold/deploy/kubectl", "CreateManifestList") manifests, err = createManifestList(k.hydratedManifests) if err != nil { return nil, err } manifests, err = manifests.SetLabels(k.labels) + endTrace() case k.skipRender: + ctx, endTrace := instrumentation.StartTrace(ctx, "pkg/skaffold/deploy/kubectl", "ReadManifests") manifests, err = k.readManifests(ctx, false) + endTrace() default: + ctx, endTrace := instrumentation.StartTrace(ctx, "pkg/skaffold/deploy/kubectl", "RenderManifests") manifests, err = k.renderManifests(ctx, out, builds, false) + endTrace() } if err != nil { @@ -118,21 +125,25 @@ func (k *Deployer) Deploy(ctx context.Context, out io.Writer, builds []graph.Art if len(manifests) == 0 { return nil, nil } - + ctx, endTrace := instrumentation.StartTrace(ctx, "pkg/skaffold/deploy/kubectl", "CollectNamespaces") namespaces, err := manifests.CollectNamespaces() if err != nil { event.DeployInfoEvent(fmt.Errorf("could not fetch deployed resource namespace. "+ "This might cause port-forward and deploy health-check to fail: %w", err)) } + endTrace() + ctx, endTrace = instrumentation.StartTrace(ctx, "pkg/skaffold/deploy/kubectl", "WaitForDeletions") if err := k.kubectl.WaitForDeletions(ctx, textio.NewPrefixWriter(out, " - "), manifests); err != nil { return nil, err } + endTrace() + ctx, endTrace = instrumentation.StartTrace(ctx, "pkg/skaffold/deploy/kubectl", "KubectlApply") if err := k.kubectl.Apply(ctx, textio.NewPrefixWriter(out, " - "), manifests); err != nil { return nil, err } - + endTrace() return namespaces, nil } @@ -308,7 +319,7 @@ func (k *Deployer) renderManifests(ctx context.Context, out io.Writer, builds [] } } - manifests, err = manifests.ReplaceImages(builds) + manifests, err = manifests.ReplaceImages(ctx, builds) if err != nil { return nil, err } @@ -340,7 +351,7 @@ func (k *Deployer) Cleanup(ctx context.Context, out io.Writer) error { rm = append(rm, manifest) } - upd, err := rm.ReplaceImages(k.originalImages) + upd, err := rm.ReplaceImages(ctx, k.originalImages) if err != nil { return err } diff --git a/pkg/skaffold/deploy/kustomize/kustomize.go b/pkg/skaffold/deploy/kustomize/kustomize.go index ff5e010cfe3..dc76682a1ec 100644 --- a/pkg/skaffold/deploy/kustomize/kustomize.go +++ b/pkg/skaffold/deploy/kustomize/kustomize.go @@ -197,7 +197,7 @@ func (k *Deployer) renderManifests(ctx context.Context, out io.Writer, builds [] return nil, nil } - manifests, err = manifests.ReplaceImages(builds) + manifests, err = manifests.ReplaceImages(ctx, builds) if err != nil { return nil, err } diff --git a/pkg/skaffold/graph/dependencies.go b/pkg/skaffold/graph/dependencies.go index cec6592b72b..b41f2e94b2a 100644 --- a/pkg/skaffold/graph/dependencies.go +++ b/pkg/skaffold/graph/dependencies.go @@ -26,7 +26,10 @@ import ( "github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/jib" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/misc" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/instrumentation" latestV1 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v1" + + // latest_v1 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v1" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/util" ) @@ -57,6 +60,10 @@ type dependencyResolverImpl struct { } func (r *dependencyResolverImpl) TransitiveArtifactDependencies(ctx context.Context, a *latestV1.Artifact) ([]string, error) { + ctx, endTrace := instrumentation.StartTrace(ctx, "pkg/skaffold/graph", "TransitiveArtifactDependencies", + instrumentation.TraceAttribute{Key: "ArtifactName", Value: a.ImageName}) + defer endTrace() + deps, err := r.SingleArtifactDependencies(ctx, a) if err != nil { return nil, err @@ -72,6 +79,10 @@ func (r *dependencyResolverImpl) TransitiveArtifactDependencies(ctx context.Cont } func (r *dependencyResolverImpl) SingleArtifactDependencies(ctx context.Context, a *latestV1.Artifact) ([]string, error) { + ctx, endTrace := instrumentation.StartTrace(ctx, "pkg/skaffold/graph", "SingleArtifactDependencies", + instrumentation.TraceAttribute{Key: "ArtifactName", Value: a.ImageName}) + defer endTrace() + res := r.cache.Exec(a.ImageName, func() interface{} { d, e := getDependenciesFunc(ctx, a, r.cfg, r.artifactResolver) if e != nil { diff --git a/pkg/skaffold/instrumentation/export.go b/pkg/skaffold/instrumentation/export.go index 507d550d679..30c938da2d3 100644 --- a/pkg/skaffold/instrumentation/export.go +++ b/pkg/skaffold/instrumentation/export.go @@ -20,23 +20,31 @@ import ( "context" "encoding/json" "fmt" + "io" "io/ioutil" "math/rand" + "net/http" "os" "path/filepath" "strconv" "time" mexporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric" + texporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace" "github.com/mitchellh/go-homedir" "github.com/rakyll/statik/fs" "github.com/sirupsen/logrus" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/exporters/stdout" + "go.opentelemetry.io/otel/exporters/trace/jaeger" "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/metric/global" "go.opentelemetry.io/otel/sdk/metric/controller/basic" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/semconv" + "go.opentelemetry.io/otel/trace" "google.golang.org/api/option" "github.com/GoogleContainerTools/skaffold/cmd/skaffold/app/cmd/statik" @@ -96,24 +104,33 @@ func exportMetrics(ctx context.Context, filename string, meter skaffoldMeter) er return nil } -func initCloudMonitoringExporterMetrics() (*basic.Controller, error) { - statikFS, err := statik.FS() - if err != nil { - return nil, err - } - b, err := fs.ReadFile(statikFS, "/secret/keys.json") +func getProjectAuthFromKey(fileFS http.FileSystem, path string) ([]byte, string, error) { + b, err := fs.ReadFile(fileFS, path) if err != nil { - // No keys have been set in this version so do not attempt to write metrics - if os.IsNotExist(err) { - return devStdOutExporter() - } - return nil, err + return nil, "", err } var c creds err = json.Unmarshal(b, &c) if c.ProjectID == "" || err != nil { - return nil, fmt.Errorf("no project id found in metrics credentials") + return nil, "", fmt.Errorf("no project id found in metrics credentials") + } + + return b, c.ProjectID, nil +} + +func initCloudMonitoringExporterMetrics() (*basic.Controller, error) { + statikFS, err := statik.FS() + if err != nil { + return nil, err + } + b, projectID, err := getProjectAuthFromKey(statikFS, "/secret/keys.json") + // No keys have been set in this version so do not attempt to write metrics + if os.IsNotExist(err) { + _, controller, err := devStdOutExporter() + return controller, err + } else if err != nil { + return nil, err } formatter := func(desc *metric.Descriptor) string { @@ -123,7 +140,7 @@ func initCloudMonitoringExporterMetrics() (*basic.Controller, error) { otel.SetErrorHandler(errHandler{}) return mexporter.InstallNewPipeline( []mexporter.Option{ - mexporter.WithProjectID(c.ProjectID), + mexporter.WithProjectID(projectID), mexporter.WithMetricDescriptorTypeFormatter(formatter), mexporter.WithMonitoringClientOptions(option.WithCredentialsJSON(b)), mexporter.WithOnError(func(err error) { @@ -133,16 +150,15 @@ func initCloudMonitoringExporterMetrics() (*basic.Controller, error) { ) } -func devStdOutExporter() (*basic.Controller, error) { +func devStdOutExporter() (*sdktrace.TracerProvider, *basic.Controller, error) { // export metrics to std out if local env is set. if _, ok := os.LookupEnv("SKAFFOLD_EXPORT_TO_STDOUT"); ok { - _, controller, err := stdout.InstallNewPipeline([]stdout.Option{ + return stdout.InstallNewPipeline([]stdout.Option{ stdout.WithPrettyPrint(), stdout.WithWriter(os.Stdout), }, nil) - return controller, err } - return nil, nil + return nil, nil, nil } func createMetrics(ctx context.Context, meter skaffoldMeter) { @@ -283,3 +299,160 @@ func errorMetrics(ctx context.Context, meter skaffoldMeter, m metric.Meter, labe unknownCounter.Record(ctx, 1, labels...) } } + +type TraceExporterConfig struct { + writer io.Writer + keyFS http.FileSystem + keyPath string + jaegerURL string +} + +type TraceExporterOption func(te *TraceExporterConfig) + +func WithWriter(w io.Writer) TraceExporterOption { + return func(teconf *TraceExporterConfig) { + teconf.writer = w + } +} + +func WithKeyFS(keyFS http.FileSystem) TraceExporterOption { + return func(teconf *TraceExporterConfig) { + teconf.keyFS = keyFS + } +} + +func WithJaegerURL(jaegerURL string) TraceExporterOption { + return func(teconf *TraceExporterConfig) { + teconf.jaegerURL = jaegerURL + } +} + +func WithKeyPath(keyPath string) TraceExporterOption { + return func(teconf *TraceExporterConfig) { + teconf.keyPath = keyPath + } +} + +func initTraceExporter(opts ...TraceExporterOption) (trace.TracerProvider, func(context.Context) error, error) { + teconf := TraceExporterConfig{ + writer: os.Stdout, + keyFS: nil, + keyPath: "/secret/keys.json", + jaegerURL: "http://localhost:14268/api/traces", + } + + for _, opt := range opts { + opt(&teconf) + } + + if _, ok := os.LookupEnv("SKAFFOLD_EXPORT_TO_STDOUT"); ok { + logrus.Debugf("using stdout trace exporter") + return initIOWriterTracer(teconf.writer) + } + + switch os.Getenv("SKAFFOLD_TRACE") { + case "stdout": + logrus.Debugf("using stdout trace exporter") + return initIOWriterTracer(teconf.writer) + case "gcp-skaffold": + logrus.Debugf("using cloud trace exporter w/ skaffold creds") + if teconf.keyFS == nil { + statikFS, err := statik.FS() + if err != nil { + return nil, func(context.Context) error { return nil }, err + } + teconf.keyFS = statikFS + } + return initCloudTraceExporterSkaffold(teconf.keyFS, teconf.keyPath) + case "gcp-adc": + logrus.Debugf("using cloud trace exporter w/ application default creds") + tp, shutdown, err := initCloudTraceExporterApplicationDefaultCreds() + return tp, func(context.Context) error { shutdown(); return nil }, err + case "jaeger": + logrus.Debugf("using jaeger trace exporter") + tp, shutdown, err := initJaegerTraceExporter(teconf.jaegerURL) + return tp, func(context.Context) error { shutdown(); return nil }, err + default: + return nil, func(context.Context) error { return nil }, fmt.Errorf("error initializing trace exporter") + } +} + +// initIOWriterTracer creates and registers trace provider instance that writes to an io.Writer interface +func initIOWriterTracer(w io.Writer) (*sdktrace.TracerProvider, func(context.Context) error, error) { + exp, err := stdout.NewExporter( + stdout.WithWriter(w), + stdout.WithPrettyPrint(), + ) + if err != nil { + return nil, func(context.Context) error { return nil }, err + } + bsp := sdktrace.NewBatchSpanProcessor(exp) + tp := sdktrace.NewTracerProvider( + sdktrace.WithSampler(sdktrace.AlwaysSample()), + sdktrace.WithSpanProcessor(bsp), + ) + return tp, tp.Shutdown, nil +} + +func initCloudTraceExporterSkaffold(keyFS http.FileSystem, keyPath string) (trace.TracerProvider, func(context.Context) error, error) { + b, projectID, err := getProjectAuthFromKey(keyFS, keyPath) + // No keys have been set in this version so do not attempt to write metrics + if os.IsNotExist(err) { + if _, ok := os.LookupEnv("SKAFFOLD_EXPORT_TO_STDOUT"); ok { + return initIOWriterTracer(os.Stdout) + } + return nil, func(context.Context) error { return nil }, nil + } else if err != nil { + return nil, func(context.Context) error { return nil }, err + } + + otel.SetErrorHandler(errHandler{}) + tp, shutdown, err := texporter.InstallNewPipeline( + []texporter.Option{ + texporter.WithProjectID(projectID), + texporter.WithTraceClientOptions([]option.ClientOption{option.WithCredentialsJSON(b)}), + texporter.WithOnError(func(err error) { + logrus.Debugf("Error with metrics: %v", err) + }), + }, + // TODO(aaron-prindle) find sane sampling to use for sporadic collection + sdktrace.WithSampler(sdktrace.AlwaysSample()), + ) + return tp, func(context.Context) error { shutdown(); return nil }, err +} + +func initCloudTraceExporterApplicationDefaultCreds() (trace.TracerProvider, func(), error) { + otel.SetErrorHandler(errHandler{}) + return texporter.InstallNewPipeline( + []texporter.Option{ + texporter.WithProjectID(os.Getenv("GOOGLE_CLOUD_PROJECT")), + texporter.WithOnError(func(err error) { + logrus.Debugf("Error with metrics: %v", err) + }), + }, + sdktrace.WithSampler(sdktrace.AlwaysSample()), + ) +} + +// initJaegerTraceExporter returns an OpenTelemetry TracerProvider configured to use +// the Jaeger exporter that will send spans to the provided url. The returned +// TracerProvider will also use a Resource configured with all the information +// about the application. +func initJaegerTraceExporter(url string) (trace.TracerProvider, func(), error) { + // Create the Jaeger exporter + exp, err := jaeger.NewRawExporter(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(url))) + if err != nil { + return nil, func() {}, err + } + tp := sdktrace.NewTracerProvider( + // Always be sure to batch in production. + sdktrace.WithBatcher(exp), + sdktrace.WithSampler(sdktrace.AlwaysSample()), + // Record information about this application in an Resource. + sdktrace.WithResource(resource.NewWithAttributes( + semconv.ServiceNameKey.String("skaffold-trace"), + attribute.Int64("ID", 1), // TODO(aaron-prindle) verify this value makes sense + )), + ) + return tp, func() {}, nil +} diff --git a/pkg/skaffold/instrumentation/export_test.go b/pkg/skaffold/instrumentation/export_test.go index 91df8f8d530..264b51d04b9 100644 --- a/pkg/skaffold/instrumentation/export_test.go +++ b/pkg/skaffold/instrumentation/export_test.go @@ -209,7 +209,7 @@ func TestInitCloudMonitoring(t *testing.T) { }, }, { - name: "key not present returns nill err", + name: "key not present returns nil err", fileSystem: &testutil.FakeFileSystem{ Files: map[string][]byte{}, }, diff --git a/pkg/skaffold/instrumentation/trace.go b/pkg/skaffold/instrumentation/trace.go new file mode 100644 index 00000000000..aab2b0e8a6e --- /dev/null +++ b/pkg/skaffold/instrumentation/trace.go @@ -0,0 +1,74 @@ +/* +Copyright 2020 The Skaffold Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package instrumentation + +import ( + "context" + "os" + "sync" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +var traceEnabled bool +var traceInitOnce sync.Once + +var tracerProvider trace.TracerProvider +var tracerShutdown func(context.Context) error +var tracerInitErr error + +func InitTraceFromEnvVar(opts ...TraceExporterOption) (trace.TracerProvider, func(context.Context) error, error) { + traceInitOnce.Do(func() { + _, skaffTrace := os.LookupEnv("SKAFFOLD_TRACE") + _, skaffExportToStdout := os.LookupEnv("SKAFFOLD_EXPORT_TO_STDOUT") + if skaffTrace || skaffExportToStdout { + traceEnabled = true + } + if traceEnabled { + tp, shutdown, err := initTraceExporter(opts...) + otel.SetTracerProvider(tp) + tracerProvider = tp + tracerShutdown = shutdown + tracerInitErr = err + } + }) + return tracerProvider, tracerShutdown, tracerInitErr +} + +func TracerShutdown(ctx context.Context) error { + traceInitOnce = sync.Once{} + return tracerShutdown(ctx) +} + +type TraceAttribute struct { + Key string + Value string +} + +func StartTrace(ctx context.Context, pkg, name string, attributes ...TraceAttribute) (context.Context, func(options ...trace.SpanOption)) { + if traceEnabled { + tracer := otel.Tracer(pkg) + ctx, span := tracer.Start(ctx, name) + for _, tr := range attributes { + span.SetAttributes(attribute.Key(tr.Key).String(tr.Value)) + } + return ctx, span.End + } + return ctx, func(options ...trace.SpanOption) {} +} diff --git a/pkg/skaffold/instrumentation/trace_test.go b/pkg/skaffold/instrumentation/trace_test.go new file mode 100644 index 00000000000..cd3cd35eeb0 --- /dev/null +++ b/pkg/skaffold/instrumentation/trace_test.go @@ -0,0 +1,159 @@ +/* +Copyright 2021 The Skaffold Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package instrumentation + +import ( + "bytes" + "context" + "encoding/json" + "os" + "testing" + + "github.com/GoogleContainerTools/skaffold/testutil" +) + +func TestInitCloudTrace(t *testing.T) { + tests := []struct { + shouldError bool + traceProviderIsNil bool + name string + traceEnvVar string + stdoutEnvVar string + spans []string + fileSystem *testutil.FakeFileSystem + }{ + { + name: "SKAFFOLD_TRACE=gcp-skaffold and key present, trace provider is not nil", + fileSystem: &testutil.FakeFileSystem{ + Files: map[string][]byte{"/secret/keys.json": []byte(testKey)}, + }, + traceEnvVar: "gcp-skaffold", + }, + { + name: "SKAFFOLD_TRACE=gcp-skaffold and key not present, returns nil err", + fileSystem: &testutil.FakeFileSystem{ + Files: map[string][]byte{}, + }, + traceEnvVar: "gcp-skaffold", + traceProviderIsNil: true, + }, + { + name: "SKAFFOLD_TRACE=gcp-skaffold and key present, credentials without project_id returns an error", + fileSystem: &testutil.FakeFileSystem{ + Files: map[string][]byte{ + "/secret/keys.json": []byte(`{ + "client_id": "test_id", + "client_secret": "test_secret", + "refresh_token": "test_token", + "type": "authorized_user" + }`, + )}, + }, + traceEnvVar: "gcp-skaffold", + shouldError: true, + traceProviderIsNil: true, + }, + { + name: "SKAFFOLD_TRACE=stdout, verify all spans output to stdout", + traceEnvVar: "stdout", + spans: []string{"SpanOne", "SpanTwo"}, + }, + { + name: "SKAFFOLD_EXPORT_TO_STDOUT=true, verify all spans output to stdout", + stdoutEnvVar: "true", + spans: []string{"SpanOne", "SpanTwo"}, + }, + } + for _, test := range tests { + testutil.Run(t, test.name, func(t *testutil.T) { + if len(test.traceEnvVar) > 0 { + os.Setenv("SKAFFOLD_TRACE", test.traceEnvVar) + defer os.Unsetenv(("SKAFFOLD_TRACE")) + } + if len(test.stdoutEnvVar) > 0 { + os.Setenv("SKAFFOLD_EXPORT_TO_STDOUT", test.stdoutEnvVar) + defer os.Unsetenv(("SKAFFOLD_EXPORT_TO_STDOUT")) + } + var b bytes.Buffer + func() { + ctx := context.Background() + tp, _, err := InitTraceFromEnvVar(WithWriter(&b), WithKeyFS(test.fileSystem)) + t.CheckErrorAndDeepEqual(test.shouldError, err, test.traceProviderIsNil || test.shouldError, tp == nil) + defer func() { _ = TracerShutdown(ctx) }() + + for _, name := range test.spans { + _, endTrace := StartTrace(ctx, "trace-test", name) + endTrace() + } + }() + if len(test.spans) > 0 { + var spans SpanArray + err := json.Unmarshal(b.Bytes(), &spans) + if err != nil { + t.Errorf("unexpected error occurred unmarshalling trace spans %v: %v", b.String(), err) + } + t.CheckTrue(len(spans) == len(test.spans)) + } + }) + } +} + +type SpanArray []struct { + Spancontext Spancontext `json:"SpanContext"` + Parent Parent `json:"Parent"` + Spankind int `json:"SpanKind"` + Name string `json:"Name"` + Starttime string `json:"StartTime"` + Endtime string `json:"EndTime"` + Attributes interface{} `json:"Attributes"` + Messageevents interface{} `json:"MessageEvents"` + Links interface{} `json:"Links"` + Statuscode string `json:"StatusCode"` + Statusmessage string `json:"StatusMessage"` + Droppedattributecount int `json:"DroppedAttributeCount"` + Droppedmessageeventcount int `json:"DroppedMessageEventCount"` + Droppedlinkcount int `json:"DroppedLinkCount"` + Childspancount int `json:"ChildSpanCount"` + Resource []Resource `json:"Resource"` + Instrumentationlibrary Instrumentationlibrary `json:"InstrumentationLibrary"` +} +type Spancontext struct { + Traceid string `json:"TraceID"` + Spanid string `json:"SpanID"` + Traceflags string `json:"TraceFlags"` + Tracestate interface{} `json:"TraceState"` + Remote bool `json:"Remote"` +} +type Parent struct { + Traceid string `json:"TraceID"` + Spanid string `json:"SpanID"` + Traceflags string `json:"TraceFlags"` + Tracestate interface{} `json:"TraceState"` + Remote bool `json:"Remote"` +} +type Value struct { + Type string `json:"Type"` + Value string `json:"Value"` +} +type Resource struct { + Key string `json:"Key"` + Value Value `json:"Value"` +} +type Instrumentationlibrary struct { + Name string `json:"Name"` + Version string `json:"Version"` +} diff --git a/pkg/skaffold/kubernetes/manifest/images.go b/pkg/skaffold/kubernetes/manifest/images.go index b3602b31620..28aeef6e63f 100644 --- a/pkg/skaffold/kubernetes/manifest/images.go +++ b/pkg/skaffold/kubernetes/manifest/images.go @@ -17,10 +17,13 @@ limitations under the License. package manifest import ( + "context" + "github.com/sirupsen/logrus" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/graph" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/instrumentation" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/warnings" ) @@ -58,7 +61,10 @@ func (is *imageSaver) Visit(o map[string]interface{}, k string, v interface{}) b } // ReplaceImages replaces image names in a list of manifests. -func (l *ManifestList) ReplaceImages(builds []graph.Artifact) (ManifestList, error) { +func (l *ManifestList) ReplaceImages(ctx context.Context, builds []graph.Artifact) (ManifestList, error) { + _, endTrace := instrumentation.StartTrace(ctx, "pkg/skaffold/kubernetes/manifest", "ReplaceImages") + defer endTrace() + replacer := newImageReplacer(builds) updated, err := l.Visit(replacer) diff --git a/pkg/skaffold/kubernetes/manifest/images_test.go b/pkg/skaffold/kubernetes/manifest/images_test.go index 9bc835ac3d3..8b7109ff2ac 100644 --- a/pkg/skaffold/kubernetes/manifest/images_test.go +++ b/pkg/skaffold/kubernetes/manifest/images_test.go @@ -17,6 +17,7 @@ limitations under the License. package manifest import ( + "context" "testing" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/graph" @@ -126,7 +127,7 @@ spec: fakeWarner := &warnings.Collect{} t.Override(&warnings.Printf, fakeWarner.Warnf) - resultManifest, err := manifests.ReplaceImages(builds) + resultManifest, err := manifests.ReplaceImages(context.TODO(), builds) t.CheckNoError(err) t.CheckDeepEqual(expected.String(), resultManifest.String()) @@ -140,7 +141,7 @@ func TestReplaceEmptyManifest(t *testing.T) { manifests := ManifestList{[]byte(""), []byte(" ")} expected := ManifestList{} - resultManifest, err := manifests.ReplaceImages(nil) + resultManifest, err := manifests.ReplaceImages(context.TODO(), nil) testutil.CheckErrorAndDeepEqual(t, false, err, expected.String(), resultManifest.String()) } @@ -148,7 +149,7 @@ func TestReplaceEmptyManifest(t *testing.T) { func TestReplaceInvalidManifest(t *testing.T) { manifests := ManifestList{[]byte("INVALID")} - _, err := manifests.ReplaceImages(nil) + _, err := manifests.ReplaceImages(context.TODO(), nil) testutil.CheckError(t, true, err) } @@ -161,7 +162,7 @@ image: - value2 `)} - output, err := manifests.ReplaceImages(nil) + output, err := manifests.ReplaceImages(context.TODO(), nil) testutil.CheckErrorAndDeepEqual(t, false, err, manifests.String(), output.String()) } diff --git a/pkg/skaffold/kubernetes/portforward/forwarder_manager.go b/pkg/skaffold/kubernetes/portforward/forwarder_manager.go index b127300cac3..d30f5d7eb34 100644 --- a/pkg/skaffold/kubernetes/portforward/forwarder_manager.go +++ b/pkg/skaffold/kubernetes/portforward/forwarder_manager.go @@ -28,6 +28,7 @@ import ( "github.com/GoogleContainerTools/skaffold/pkg/skaffold/constants" debugging "github.com/GoogleContainerTools/skaffold/pkg/skaffold/debug" eventV2 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/event/v2" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/instrumentation" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubectl" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes" latestV1 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v1" @@ -118,6 +119,10 @@ func (f *ForwarderManager) Start(ctx context.Context, out io.Writer, namespaces } eventV2.TaskInProgress(constants.PortForward) + + ctx, endTrace := instrumentation.StartTrace(ctx, "pkg/skaffold/kubernetes/portforward", "PortForward") + defer endTrace() + for _, f := range f.forwarders { if err := f.Start(ctx, out, namespaces); err != nil { eventV2.TaskFailed(constants.PortForward, err) diff --git a/pkg/skaffold/runner/v1/apply.go b/pkg/skaffold/runner/v1/apply.go index 75059f24bad..2508e47e56b 100644 --- a/pkg/skaffold/runner/v1/apply.go +++ b/pkg/skaffold/runner/v1/apply.go @@ -25,6 +25,7 @@ import ( deployutil "github.com/GoogleContainerTools/skaffold/pkg/skaffold/deploy/util" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/event" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/graph" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/instrumentation" ) // Apply sends Kubernetes manifests to the cluster. @@ -63,6 +64,8 @@ func (r *SkaffoldRunner) applyResources(ctx context.Context, out io.Writer, arti } event.DeployInProgress() + ctx, endTrace := instrumentation.StartTrace(ctx, "pkg/skaffold/runner", "DeployInProgress") + defer endTrace() namespaces, err := r.deployer.Deploy(ctx, deployOut, artifacts) postDeployFn() if err != nil { diff --git a/pkg/skaffold/runner/v1/deploy.go b/pkg/skaffold/runner/v1/deploy.go index 2c746c5cb82..9143f27a013 100644 --- a/pkg/skaffold/runner/v1/deploy.go +++ b/pkg/skaffold/runner/v1/deploy.go @@ -32,6 +32,7 @@ import ( "github.com/GoogleContainerTools/skaffold/pkg/skaffold/event" eventV2 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/event/v2" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/graph" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/instrumentation" kubernetesclient "github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes/client" kubectx "github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes/context" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/runner" @@ -124,6 +125,9 @@ See https://skaffold.dev/docs/pipeline-stages/taggers/#how-tagging-works`) event.DeployInProgress() eventV2.TaskInProgress(constants.Deploy) + ctx, endTrace := instrumentation.StartTrace(ctx, "pkg/skaffold/runner", "DeployInProgress") + defer endTrace() + namespaces, err := r.deployer.Deploy(ctx, deployOut, artifacts) postDeployFn() if err != nil { @@ -139,6 +143,7 @@ See https://skaffold.dev/docs/pipeline-stages/taggers/#how-tagging-works`) if err != nil { return err } + event.DeployComplete() eventV2.TaskSucceeded(constants.Deploy) r.runCtx.UpdateNamespaces(namespaces) @@ -209,6 +214,9 @@ func (r *SkaffoldRunner) performStatusCheck(ctx context.Context, out io.Writer) } eventV2.TaskInProgress(constants.StatusCheck) + ctx, endTrace := instrumentation.StartTrace(ctx, "pkg/skaffold/runner", "WaitForDeploymentToStabilize") + defer endTrace() + start := time.Now() color.Default.Fprintln(out, "Waiting for deployments to stabilize...") diff --git a/pkg/skaffold/runner/v1/dev.go b/pkg/skaffold/runner/v1/dev.go index 726e0649040..903ff615a9d 100644 --- a/pkg/skaffold/runner/v1/dev.go +++ b/pkg/skaffold/runner/v1/dev.go @@ -67,9 +67,11 @@ func (r *SkaffoldRunner) doDev(ctx context.Context, out io.Writer) error { // if any action is going to be performed, reset the monitor's changed component tracker for debouncing defer r.monitor.Reset() defer r.listener.LogWatchToUser(out) + event.DevLoopInProgress(r.devIteration) eventV2.TaskInProgress(constants.DevLoop) defer func() { r.devIteration++ }() + ctx, endTrace := instrumentation.StartTrace(ctx, "pkg/skaffold/runner", "DevLoopInProgress") meterUpdated := false if needsSync { @@ -167,6 +169,8 @@ func (r *SkaffoldRunner) doDev(ctx context.Context, out io.Writer) error { } event.DevLoopComplete(r.devIteration) eventV2.TaskSucceeded(constants.DevLoop) + + endTrace() r.deployer.Unmute() return nil } @@ -177,6 +181,8 @@ func (r *SkaffoldRunner) Dev(ctx context.Context, out io.Writer, artifacts []*la event.DevLoopInProgress(r.devIteration) eventV2.TaskInProgress(constants.DevLoop) defer func() { r.devIteration++ }() + ctx, endTrace := instrumentation.StartTrace(ctx, "pkg/skaffold/runner", "DevLoopInProgress") + g := getTransposeGraph(artifacts) // Watch artifacts start := time.Now() @@ -307,6 +313,7 @@ func (r *SkaffoldRunner) Dev(ctx context.Context, out io.Writer, artifacts []*la event.DevLoopComplete(r.devIteration) eventV2.TaskSucceeded(constants.DevLoop) + endTrace() r.devIteration++ return r.listener.WatchForChanges(ctx, out, func() error { return r.doDev(ctx, out) diff --git a/pkg/skaffold/runner/v1/new.go b/pkg/skaffold/runner/v1/new.go index 33841c0ab95..c3153a793e3 100644 --- a/pkg/skaffold/runner/v1/new.go +++ b/pkg/skaffold/runner/v1/new.go @@ -37,6 +37,7 @@ import ( eventV2 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/event/v2" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/filemon" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/graph" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/instrumentation" pkgkubectl "github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubectl" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes/logger" @@ -59,6 +60,8 @@ func NewForConfig(runCtx *runcontext.RunContext) (*SkaffoldRunner, error) { event.LogMetaEvent() eventV2.InitializeState(runCtx) kubectlCLI := pkgkubectl.NewCLI(runCtx, "") + _, endTrace := instrumentation.StartTrace(context.Background(), "pkg/skaffold/runner", "NewForConfig") + defer endTrace() tagger, err := tag.NewTaggerMux(runCtx) if err != nil { @@ -98,6 +101,9 @@ func NewForConfig(runCtx *runcontext.RunContext) (*SkaffoldRunner, error) { } depLister := func(ctx context.Context, artifact *latestV1.Artifact) ([]string, error) { + ctx, endTrace := instrumentation.StartTrace(ctx, "pkg/skaffold/runner", "GetAllDependencies") + defer endTrace() + buildDependencies, err := sourceDependencies.SingleArtifactDependencies(ctx, artifact) if err != nil { return nil, err @@ -107,7 +113,6 @@ func NewForConfig(runCtx *runcontext.RunContext) (*SkaffoldRunner, error) { if err != nil { return nil, err } - return append(buildDependencies, testDependencies...), nil }