Expose controller-runtime metrics (#786)

* pkg/scaffold/cmd.go: Skip erroring out on Service

If the Service could not be created we just log the error instead of
erroring out and exiting. This error could occur when there is more than
one pods such as with leader election. In that case the new pod will
expose the metrics via the already created Service object.

* pkg/*: Create new client for Service get/create

Due to the cache not being started at the time when we attempt to query
for the Service, we instead create a new client in a similar way as we
do with leader elections.

* CHANGELOG: Document metrics addition

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ### Added
 
+- By default the controller-runtime metrics are exposed on port 8383. This is done as part of the scaffold in the main.go file, the port can be adjusted by modifying the `metricsPort` variable. [#786](https://github.com/operator-framework/operator-sdk/pull/786)
+
 ### Changed
 
 ### Deprecated

Gopkg.toml

@@ -26,6 +26,10 @@
   name = "k8s.io/cli-runtime"
   version = "kubernetes-1.12.3"
 
+[[override]]
+   name = "k8s.io/kube-openapi"
+  revision = "0cf8f7e6ed1d2e3d47d02e3b6e559369af24d803"
+
 [[constraint]]
   name = "sigs.k8s.io/controller-runtime"
   version = "=v0.1.8"

commands/operator-sdk/cmd/build_test.go

@@ -90,9 +90,6 @@ spec:
       containers:
         - name: memcached-operator
           image: quay.io/coreos/operator-sdk-dev:test-framework-operator
-          ports:
-          - containerPort: 60000
-            name: metrics
           command:
           - memcached-operator
           imagePullPolicy: Always

pkg/k8sutil/constants.go

@@ -27,10 +27,4 @@ const (
 	// OperatorNameEnvVar is the constant for env variable OPERATOR_NAME
 	// wich is the name of the current operator
 	OperatorNameEnvVar = "OPERATOR_NAME"
-
-	// PrometheusMetricsPort defines the port which expose prometheus metrics
-	PrometheusMetricsPort = 60000
-
-	// PrometheusMetricsPortName define the port name used in kubernetes deployment and service
-	PrometheusMetricsPortName = "metrics"
 )

pkg/k8sutil/k8sutil.go

@@ -20,9 +20,6 @@ import (
 	"os"
 	"strings"
 
-	v1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	intstr "k8s.io/apimachinery/pkg/util/intstr"
 	discovery "k8s.io/client-go/discovery"
 	logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
 )
@@ -68,44 +65,6 @@ func GetOperatorName() (string, error) {
 	return operatorName, nil
 }
 
-// InitOperatorService return the static service which expose operator metrics
-func InitOperatorService() (*v1.Service, error) {
-	operatorName, err := GetOperatorName()
-	if err != nil {
-		return nil, err
-	}
-	namespace, err := GetOperatorNamespace()
-	if err != nil {
-		return nil, err
-	}
-	service := &v1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      operatorName,
-			Namespace: namespace,
-			Labels:    map[string]string{"name": operatorName},
-		},
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: "v1",
-		},
-		Spec: v1.ServiceSpec{
-			Ports: []v1.ServicePort{
-				{
-					Port:     PrometheusMetricsPort,
-					Protocol: v1.ProtocolTCP,
-					TargetPort: intstr.IntOrString{
-						Type:   intstr.String,
-						StrVal: PrometheusMetricsPortName,
-					},
-					Name: PrometheusMetricsPortName,
-				},
-			},
-			Selector: map[string]string{"name": operatorName},
-		},
-	}
-	return service, nil
-}
-
 // ResourceExists returns true if the given resource kind exists
 // in the given api groupversion
 func ResourceExists(dc discovery.DiscoveryInterface, apiGroupVersion, kind string) (bool, error) {

pkg/metrics/metrics.go

@@ -16,45 +16,111 @@ package metrics
 
 import (
 	"context"
-	"net/http"
-	"strconv"
+	"fmt"
 
 	"github.com/operator-framework/operator-sdk/pkg/k8sutil"
 
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	v1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/errors"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/client/config"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/client-go/rest"
+	crclient "sigs.k8s.io/controller-runtime/pkg/client"
 	logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
 )
 
 var log = logf.Log.WithName("metrics")
 
-// ExposeMetricsPort generate a Kubernetes Service to expose metrics port
-func ExposeMetricsPort() *v1.Service {
-	http.Handle("/"+k8sutil.PrometheusMetricsPortName, promhttp.Handler())
-	go http.ListenAndServe(":"+strconv.Itoa(k8sutil.PrometheusMetricsPort), nil)
+// PrometheusPortName defines the port name used in kubernetes deployment and service resources
+const PrometheusPortName = "metrics"
 
-	service, err := k8sutil.InitOperatorService()
+// ExposeMetricsPort creates a Kubernetes Service to expose the passed metrics port.
+func ExposeMetricsPort(ctx context.Context, port int32) (*v1.Service, error) {
+	// We do not need to check the validity of the port, as controller-runtime
+	// would error out and we would never get to this stage.
+	s, err := initOperatorService(port, PrometheusPortName)
 	if err != nil {
-		log.Error(err, "Failed to initialize service object for operator metrics")
-		return nil
+		if err == k8sutil.ErrNoNamespace {
+			log.Info("Skipping metrics Service creation; not running in a cluster.")
+			return nil, nil
+		}
+		return nil, fmt.Errorf("failed to initialize service object for metrics: %v", err)
 	}
-	kubeconfig, err := config.GetConfig()
+	service, err := createService(ctx, s)
 	if err != nil {
-		panic(err)
+		return nil, fmt.Errorf("failed to create or get service for metrics: %v", err)
 	}
-	runtimeClient, err := client.New(kubeconfig, client.Options{})
+
+	return service, nil
+}
+
+func createService(ctx context.Context, s *v1.Service) (*v1.Service, error) {
+	config, err := rest.InClusterConfig()
 	if err != nil {
-		panic(err)
+		return nil, err
 	}
-	err = runtimeClient.Create(context.TODO(), service)
-	if err != nil && !errors.IsAlreadyExists(err) {
-		log.Error(err, "Failed to create service for operator metrics")
-		return nil
+
+	client, err := crclient.New(config, crclient.Options{})
+	if err != nil {
+		return nil, err
+	}
+
+	if err := client.Create(ctx, s); err != nil {
+		if !apierrors.IsAlreadyExists(err) {
+			return nil, err
+		}
+		// Get existing Service and return it
+		existingService := &v1.Service{}
+		err := client.Get(ctx, types.NamespacedName{
+			Name:      s.Name,
+			Namespace: s.Namespace,
+		}, existingService)
+		if err != nil {
+			return nil, err
+		}
+		log.Info("Metrics Service object already exists", "name", existingService.Name)
+		return existingService, nil
 	}
 
-	log.Info("Metrics service created.", "ServiceName", service.Name)
-	return service
+	log.Info("Metrics Service object created", "name", s.Name)
+	return s, nil
+}
+
+// initOperatorService returns the static service which exposes specifed port.
+func initOperatorService(port int32, portName string) (*v1.Service, error) {
+	operatorName, err := k8sutil.GetOperatorName()
+	if err != nil {
+		return nil, err
+	}
+	namespace, err := k8sutil.GetOperatorNamespace()
+	if err != nil {
+		return nil, err
+	}
+	service := &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      operatorName,
+			Namespace: namespace,
+			Labels:    map[string]string{"name": operatorName},
+		},
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Service",
+			APIVersion: "v1",
+		},
+		Spec: v1.ServiceSpec{
+			Ports: []v1.ServicePort{
+				{
+					Port:     port,
+					Protocol: v1.ProtocolTCP,
+					TargetPort: intstr.IntOrString{
+						Type:   intstr.Int,
+						IntVal: port,
+					},
+					Name: portName,
+				},
+			},
+			Selector: map[string]string{"name": operatorName},
+		},
+	}
+	return service, nil
 }

pkg/scaffold/ansible/deploy_operator.go

@@ -57,9 +57,6 @@ spec:
         - name: {{.ProjectName}}
           # Replace this with the built image name
           image: "{{ "{{ REPLACE_IMAGE }}" }}"
-          ports:
-          - containerPort: 60000
-            name: metrics
           imagePullPolicy: "{{ "{{ pull_policy|default('Always') }}"}}"
           env:
             - name: WATCH_NAMESPACE

pkg/scaffold/cmd.go

@@ -48,6 +48,7 @@ import (
 
 	"github.com/operator-framework/operator-sdk/pkg/k8sutil"
 	"github.com/operator-framework/operator-sdk/pkg/leader"
+	"github.com/operator-framework/operator-sdk/pkg/metrics"
 	sdkVersion "github.com/operator-framework/operator-sdk/version"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
@@ -56,6 +57,11 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/runtime/signals"
 )
 
+// Change below variables to serve metrics on different host or port.
+var (
+    metricsHost = "0.0.0.0"
+    metricsPort int32 = 8383
+)
 var log = logf.Log.WithName("cmd")
 
 func printVersion() {
@@ -87,16 +93,21 @@ func main() {
 		log.Error(err, "")
 		os.Exit(1)
 	}
+	
+	ctx := context.TODO()
 
 	// Become the leader before proceeding
-	err = leader.Become(context.TODO(), "{{ .ProjectName }}-lock")
+	err = leader.Become(ctx, "{{ .ProjectName }}-lock")
 	if err != nil {
 		log.Error(err, "")
 		os.Exit(1)
 	}
 
 	// Create a new Cmd to provide shared dependencies and start components
-	mgr, err := manager.New(cfg, manager.Options{Namespace: namespace})
+	mgr, err := manager.New(cfg, manager.Options{
+		Namespace:          namespace,
+		MetricsBindAddress: fmt.Sprintf("%s:%d", metricsHost, metricsPort),
+	})	
 	if err != nil {
 		log.Error(err, "")
 		os.Exit(1)
@@ -116,6 +127,12 @@ func main() {
 		os.Exit(1)
 	}
 
+	// Create Service object to expose the metrics port.
+	_, err = metrics.ExposeMetricsPort(ctx, metricsPort)
+	if err != nil {
+		log.Info(err.Error())
+	}
+
 	log.Info("Starting the Cmd.")
 
 	// Start the Cmd

pkg/scaffold/cmd_test.go

@@ -46,6 +46,7 @@ import (
 	"github.com/example-inc/app-operator/pkg/controller"
 	"github.com/operator-framework/operator-sdk/pkg/k8sutil"
 	"github.com/operator-framework/operator-sdk/pkg/leader"
+	"github.com/operator-framework/operator-sdk/pkg/metrics"
 	sdkVersion "github.com/operator-framework/operator-sdk/version"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
@@ -54,6 +55,11 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/runtime/signals"
 )
 
+// Change below variables to serve metrics on different host or port.
+var (
+	metricsHost       = "0.0.0.0"
+	metricsPort int32 = 8383
+)
 var log = logf.Log.WithName("cmd")
 
 func printVersion() {
@@ -86,15 +92,20 @@ func main() {
 		os.Exit(1)
 	}
 
+	ctx := context.TODO()
+
 	// Become the leader before proceeding
-	err = leader.Become(context.TODO(), "app-operator-lock")
+	err = leader.Become(ctx, "app-operator-lock")
 	if err != nil {
 		log.Error(err, "")
 		os.Exit(1)
 	}
 
 	// Create a new Cmd to provide shared dependencies and start components
-	mgr, err := manager.New(cfg, manager.Options{Namespace: namespace})
+	mgr, err := manager.New(cfg, manager.Options{
+		Namespace:          namespace,
+		MetricsBindAddress: fmt.Sprintf("%s:%d", metricsHost, metricsPort),
+	})
 	if err != nil {
 		log.Error(err, "")
 		os.Exit(1)
@@ -114,6 +125,12 @@ func main() {
 		os.Exit(1)
 	}
 
+	// Create Service object to expose the metrics port.
+	_, err = metrics.ExposeMetricsPort(ctx, metricsPort)
+	if err != nil {
+		log.Info(err.Error())
+	}
+
 	log.Info("Starting the Cmd.")
 
 	// Start the Cmd

pkg/scaffold/gopkgtoml.go

@@ -92,6 +92,10 @@ required = [
   branch = "master" #osdk_branch_annotation
   # version = "=v0.4.0" #osdk_version_annotation
 
+[[override]]
+   name = "k8s.io/kube-openapi"
+  revision = "0cf8f7e6ed1d2e3d47d02e3b6e559369af24d803"
+
 [prune]
   go-tests = true
   non-go = true

pkg/scaffold/gopkgtoml_test.go

@@ -84,6 +84,10 @@ required = [
   branch = "master" #osdk_branch_annotation
   # version = "=v0.4.0" #osdk_version_annotation
 
+[[override]]
+   name = "k8s.io/kube-openapi"
+  revision = "0cf8f7e6ed1d2e3d47d02e3b6e559369af24d803"
+
 [prune]
   go-tests = true
   non-go = true

pkg/scaffold/operator.go

@@ -55,9 +55,6 @@ spec:
         - name: {{.ProjectName}}
           # Replace this with the built image name
           image: REPLACE_IMAGE
-          ports:
-          - containerPort: 60000
-            name: metrics
           command:
           - {{.ProjectName}}
           imagePullPolicy: Always