drop stackdriver metrics and tracing exporters (#2183)

* drop stack driver as a tracing backend

* drop stackdriver as a metrics backend

* update deps - dropping stackdriver

* fix linting issues

* drop further references to stack driver

* drop serving & eventing metric key constants
these have been moved to their respective repos
see: #608

* move source metrickeys to pkg/source

We're going to move the StatsReporter downstream eventually

related:
knative/eventing#5587
knative/eventing#5586

* fix linter

go.mod

@@ -7,7 +7,6 @@ require (
 	cloud.google.com/go/storage v1.10.0
 	contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d
 	contrib.go.opencensus.io/exporter/prometheus v0.3.0
-	contrib.go.opencensus.io/exporter/stackdriver v0.13.5
 	contrib.go.opencensus.io/exporter/zipkin v0.1.2
 	github.com/blang/semver/v4 v4.0.0
 	github.com/blendle/zapdriver v1.3.1
@@ -53,7 +52,7 @@ require (
 	golang.org/x/tools v0.1.4
 	gomodules.xyz/jsonpatch/v2 v2.2.0
 	google.golang.org/api v0.36.0
-	google.golang.org/genproto v0.0.0-20210416161957-9910b6c460de
+	google.golang.org/genproto v0.0.0-20210416161957-9910b6c460de // indirect
 	google.golang.org/grpc v1.38.0
 	google.golang.org/protobuf v1.26.0
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b

go.sum

@@ -38,8 +38,6 @@ contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d h
 contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d/go.mod h1:IshRmMJBhDfFj5Y67nVhMYTTIze91RUeT73ipWKs/GY=
 contrib.go.opencensus.io/exporter/prometheus v0.3.0 h1:08FMdJYpItzsknogU6PiiNo7XQZg/25GjH236+YCwD0=
 contrib.go.opencensus.io/exporter/prometheus v0.3.0/go.mod h1:rpCPVQKhiyH8oomWgm34ZmgIdZa8OVYO5WAIygPbBBE=
-contrib.go.opencensus.io/exporter/stackdriver v0.13.5 h1:TNaexHK16gPUoc7uzELKOU7JULqccn1NDuqUxmxSqfo=
-contrib.go.opencensus.io/exporter/stackdriver v0.13.5/go.mod h1:aXENhDJ1Y4lIg4EUaVTwzvYETVNZk10Pu26tevFKLUc=
 contrib.go.opencensus.io/exporter/zipkin v0.1.2 h1:YqE293IZrKtqPnpwDPH/lOqTWD/s3Iwabycam74JV3g=
 contrib.go.opencensus.io/exporter/zipkin v0.1.2/go.mod h1:mP5xM3rrgOjpn79MM8fZbj3gsxcuytSqtH0dxSWW1RE=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
@@ -83,8 +81,6 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj
 github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
-github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
-github.com/aws/aws-sdk-go v1.27.0 h1:0xphMHGMLBrPMfxR2AmVjZKcMEESEgWF8Kru94BNByk=
 github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
@@ -362,7 +358,6 @@ github.com/influxdata/tdigest v0.0.0-20180711151920-a7d76c6f093a/go.mod h1:9Gkys
 github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9 h1:MHTrDWmQpHq/hkq+7cw9oYAt2PqUw52TZazRA0N7PGE=
 github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
-github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -698,7 +693,6 @@ golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -841,7 +835,6 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191010075000-0337d82405ff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -898,7 +891,6 @@ google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEt
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
 google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.10.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
 google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
@@ -920,7 +912,6 @@ google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=

metrics/README.md

@@ -1,15 +1,14 @@
 # Common metrics export interfaces for Knative
 
-_Note that this directory is currently in transition. See [the Plan](#the-plan)
-for details on where this is heading._
+See [the Plan](#the-plan) for details on where this is heading.
 
 ## Current status
 
 The code currently uses OpenCensus to support exporting metrics to multiple
-backends. Currently, two backends are supported: Prometheus and Stackdriver.
+backends. Currently, two backends are supported: Prometheus and OpenCensus/OTel.
 
 Metrics export is controlled by a ConfigMap called `config-observability` which
-is a key-value map with specific values supported for each of the Stackdriver
+is a key-value map with specific values supported for each of the OpenCensus
 and Prometheus backends. Hot-reload of the ConfigMap on a running process is
 supported by directly watching (via the Kubernetes API) the
 `config-observability` object. Configuration via environment is also supported
@@ -20,9 +19,7 @@ namespace.
 
 There are currently
 [6 supported Golang exporters for OpenCensus](https://opencensus.io/exporters/supported-exporters/go/).
-At least the Stackdriver exporter causes problems/failures if started without
-access to (Google) application default credentials. It's not clear that we want
-to build all of those backends into the core of `knative.dev/pkg` and all
+We do not want to build all of those backends into the core of `knative.dev/pkg` and all
 downstream dependents, and we'd like all the code shipped in `knative.dev/pkg`
 to be able to be tested without needing any special environment setup.
 
@@ -89,19 +86,22 @@ statistics for a short period of time if not.
 
 ### Steps to reach the goal
 
-- [ ] [Add OpenCensus Agent as one of the export options](https://github.com/knative/pkg/issues/955).
-- [ ] Ensure that all tests pass in a non-Google-Cloud connected environment.
+- [x] [Add OpenCensus Agent as one of the export options](https://github.com/knative/pkg/issues/955).
+- [x] Ensure that all tests pass in a non-Google-Cloud connected environment.
       **This is true today.**
       [Ensure this on an ongoing basis.](https://github.com/knative/pkg/issues/957)
-- [ ] Google to implement OpenCensus Agent configuration to match what they are
+- [x] Google to implement OpenCensus Agent configuration to match what they are
       doing for Stackdriver now. (No public issue link because this should be in
       Google's vendor-specific configuration.)
-- [ ] Document how to configure OpenCensus/OpenTelemetry Agent + Prometheus to
+- [x] Document how to configure OpenCensus/OpenTelemetry Agent + Prometheus to
       achieve the current level of application visibility, and determine a
       long-term course for how to maintain this as a "bare minimum" supported
-      configuration.
-- [ ] Stop adding exporter features outside of the OpenCensus / OpenTelemetry
+      configuration. https://github.com/knative/docs/pull/3005
+- [x] Stop adding exporter features outside of the OpenCensus / OpenTelemetry
       export as of 0.13 release (03 March 2020). Between now and 0.13, small
       amounts of additional features can be built in to assist with the bridging
       process or to support existing products. New products should build on the
       OpenCensus Agent approach.
+- [x] Removal of the Stackdriver OpenCensus Exporter
+      https://github.com/knative/pkg/issues/2173
+- [ ] Revisit Adopting OpenTelemetry SDKs instead of OpenCensus

metrics/config.go

@@ -43,17 +43,9 @@ const (
 	// The following keys are used to configure metrics reporting.
 	// See https://github.com/knative/serving/blob/main/config/config-observability.yaml
 	// for details.
-	allowStackdriverCustomMetricsKey = "metrics.allow-stackdriver-custom-metrics"
-	collectorAddressKey              = "metrics.opencensus-address"
-	collectorSecureKey               = "metrics.opencensus-require-tls"
-	reportingPeriodKey               = "metrics.reporting-period-seconds"
-
-	// Stackdriver client configuration keys
-	stackdriverClusterNameKey           = "metrics.stackdriver-cluster-name"
-	stackdriverCustomMetricSubDomainKey = "metrics.stackdriver-custom-metrics-subdomain"
-	stackdriverGCPLocationKey           = "metrics.stackdriver-gcp-location"
-	stackdriverProjectIDKey             = "metrics.stackdriver-project-id"
-	stackdriverUseSecretKey             = "metrics.stackdriver-use-secret"
+	collectorAddressKey = "metrics.opencensus-address"
+	collectorSecureKey  = "metrics.opencensus-require-tls"
+	reportingPeriodKey  = "metrics.reporting-period-seconds"
 
 	defaultBackendEnvName = "DEFAULT_METRICS_BACKEND"
 	defaultPrometheusPort = 9090
@@ -66,14 +58,12 @@ const (
 
 var (
 	// TestOverrideBundleCount is a variable for testing to reduce the size (number of metrics) buffered before
-	// Stackdriver will send a bundled metric report. Only applies if non-zero.
+	// OpenCensus will send a bundled metric report. Only applies if non-zero.
 	TestOverrideBundleCount = 0
 )
 
 // Metrics backend "enum".
 const (
-	// stackdriver is used for Stackdriver backend
-	stackdriver metricsBackend = "stackdriver"
 	// prometheus is used for Prometheus backend
 	prometheus metricsBackend = "prometheus"
 	// openCensus is used to export to the OpenCensus Agent / Collector,
@@ -104,6 +94,7 @@ type metricsConfig struct {
 	// ---- OpenCensus specific below ----
 	// collectorAddress is the address of the collector, if not `localhost:55678`
 	collectorAddress string
+
 	// Require mutual TLS. Defaults to "false" because mutual TLS is hard to set up.
 	requireSecure bool
 
@@ -115,52 +106,6 @@ type metricsConfig struct {
 	// prometheusHost is the host where the metrics are exposed in Prometheus
 	// format. It defaults to "0.0.0.0"
 	prometheusHost string
-
-	// ---- Stackdriver specific below ----
-	// True if backendDestination equals to "stackdriver". Store this in a variable
-	// to reduce string comparison operations.
-	isStackdriverBackend bool
-	// stackdriverMetricTypePrefix is the metric domain joins component, e.g.
-	// "knative.dev/serving/activator". Store this in a variable to reduce string
-	// join operations.
-	stackdriverMetricTypePrefix string
-	// stackdriverCustomMetricTypePrefix is "custom.googleapis.com" joined with the subdomain and component.
-	// E.g., "custom.googleapis.com/<subdomain>/<component>".
-	// Store this in a variable to reduce string join operations.
-	stackdriverCustomMetricTypePrefix string
-	// stackdriverClientConfig is the metadata to configure the metrics exporter's Stackdriver client.
-	stackdriverClientConfig StackdriverClientConfig
-}
-
-// StackdriverClientConfig encapsulates the metadata required to configure a Stackdriver client.
-type StackdriverClientConfig struct {
-	// ProjectID is the stackdriver project ID to which data is uploaded.
-	// This is not necessarily the GCP project ID where the Kubernetes cluster is hosted.
-	// Required when the Kubernetes cluster is not hosted on GCE.
-	ProjectID string
-	// GCPLocation is the GCP region or zone to which data is uploaded.
-	// This is not necessarily the GCP location where the Kubernetes cluster is hosted.
-	// Required when the Kubernetes cluster is not hosted on GCE.
-	GCPLocation string
-	// ClusterName is the cluster name with which the data will be associated in Stackdriver.
-	// Required when the Kubernetes cluster is not hosted on GCE.
-	ClusterName string
-	// UseSecret is whether the credentials stored in a Kubernetes Secret should be used to
-	// authenticate with Stackdriver. The Secret name and namespace can be specified by calling
-	// metrics.SetStackdriverSecretLocation.
-	// If UseSecret is false, Google Application Default Credentials
-	// will be used (https://cloud.google.com/docs/authentication/production).
-	UseSecret bool
-}
-
-// NewStackdriverClientConfigFromMap creates a stackdriverClientConfig from the given map
-func NewStackdriverClientConfigFromMap(config map[string]string) *StackdriverClientConfig {
-	return &StackdriverClientConfig{
-		ProjectID:   config[stackdriverProjectIDKey],
-		GCPLocation: config[stackdriverGCPLocationKey],
-		ClusterName: config[stackdriverClusterNameKey],
-		UseSecret:   strings.EqualFold(config[stackdriverUseSecretKey], "true"),
-	}
 }
 
 // record applies the `ros` Options to each measurement in `mss` and then records the resulting
@@ -184,7 +129,7 @@ func (mc *metricsConfig) record(ctx context.Context, mss []stats.Measurement, ro
 	return mc.recorder(ctx, mss, ros...)
 }
 
-func createMetricsConfig(ctx context.Context, ops ExporterOptions) (*metricsConfig, error) {
+func createMetricsConfig(_ context.Context, ops ExporterOptions) (*metricsConfig, error) {
 	var mc metricsConfig
 
 	if ops.Domain == "" {
@@ -213,7 +158,7 @@ func createMetricsConfig(ctx context.Context, ops ExporterOptions) (*metricsConf
 	}
 
 	switch lb := metricsBackend(strings.ToLower(backend)); lb {
-	case stackdriver, prometheus, openCensus, none:
+	case prometheus, openCensus, none:
 		mc.backendDestination = lb
 	default:
 		return nil, fmt.Errorf("unsupported metrics backend value %q", backend)
@@ -252,16 +197,12 @@ func createMetricsConfig(ctx context.Context, ops ExporterOptions) (*metricsConf
 
 		mc.prometheusPort = pp
 		mc.prometheusHost = prometheusHost()
-	case stackdriver:
-		if err := sdinit(ctx, m, &mc, ops); err != nil {
-			return nil, err
-		}
 	}
 
 	// If reporting period is specified, use the value from the configuration.
 	// If not, set a default value based on the selected backend.
 	// Each exporter makes different promises about what the lowest supported
-	// reporting period is. For Stackdriver, this value is 1 minute.
+	// reporting period is. For OpenCensus, this value is 1 minute.
 	// For Prometheus, we will use a lower value since the exporter doesn't
 	// push anything but just responds to pull requests, and shorter durations
 	// do not really hurt the performance and we rely on the scraping configuration.
@@ -273,7 +214,7 @@ func createMetricsConfig(ctx context.Context, ops ExporterOptions) (*metricsConf
 		mc.reportingPeriod = time.Duration(repInt) * time.Second
 	} else {
 		switch mc.backendDestination {
-		case stackdriver, openCensus:
+		case openCensus:
 			mc.reportingPeriod = time.Minute
 		case prometheus:
 			mc.reportingPeriod = 5 * time.Second

metrics/config_observability.go

@@ -65,8 +65,8 @@ type ObservabilityConfig struct {
 	// EnableProbeRequestLog enables queue-proxy to write health check probe request logs.
 	EnableProbeRequestLog bool
 
-	// RequestMetricsBackend specifies the request metrics destination, e.g. Prometheus,
-	// Stackdriver. "None" disables all backends.
+	// RequestMetricsBackend specifies the request metrics destination, e.g. Prometheus.
+	// "None" disables all backends.
 	RequestMetricsBackend string
 
 	// EnableProfiling indicates whether it is allowed to retrieve runtime profiling data from

metrics/config_observability_test.go

@@ -41,15 +41,15 @@ func TestObservabilityConfiguration(t *testing.T) {
 			EnableRequestLog:       true,
 			LoggingURLTemplate:     "https://logging.io",
 			RequestLogTemplate:     `{"requestMethod": "{{.Request.Method}}"}`,
-			RequestMetricsBackend:  "stackdriver",
+			RequestMetricsBackend:  "opencensus",
 		},
 		data: map[string]string{
 			EnableProbeReqLogKey:                          "true",
 			"logging.enable-var-log-collection":           "true",
 			ReqLogTemplateKey:                             `{"requestMethod": "{{.Request.Method}}"}`,
 			"logging.revision-url-template":               "https://logging.io",
 			EnableReqLogKey:                               "true",
-			"metrics.request-metrics-backend-destination": "stackdriver",
+			"metrics.request-metrics-backend-destination": "opencensus",
 			"profiling.enable":                            "true",
 		},
 	}, {