chore: add more metrics

pkg/controller/kubefedcluster/clusterclient.go

@@ -129,13 +129,13 @@ func (self *ClusterClient) GetClusterHealthStatus() (*fedv1b1.KubeFedClusterStat
 	if err != nil {
 		runtime.HandleError(errors.Wrapf(err, "Failed to do cluster health check for cluster %q", self.clusterName))
 		clusterStatus.Conditions = append(clusterStatus.Conditions, newClusterOfflineCondition)
-		metrics.RegisterKubefedClusterOfflineCount(self.clusterName)
+		metrics.RegisterKubefedClusterTotal(metrics.ClusterOffline, self.clusterName)
 	} else {
 		if !strings.EqualFold(string(body), "ok") {
-			metrics.RegisterKubefedClusterNotReadyCount(self.clusterName)
+			metrics.RegisterKubefedClusterTotal(metrics.ClusterNotReady, self.clusterName)
 			clusterStatus.Conditions = append(clusterStatus.Conditions, newClusterNotReadyCondition, newClusterNotOfflineCondition)
 		} else {
-			metrics.RegisterKubefedClusterReadyCount(self.clusterName)
+			metrics.RegisterKubefedClusterTotal(metrics.ClusterReady, self.clusterName)
 			clusterStatus.Conditions = append(clusterStatus.Conditions, newClusterReadyCondition)
 		}
 	}

pkg/controller/kubefedcluster/controller.go

@@ -261,7 +261,7 @@ func (cc *ClusterController) updateIndividualClusterStatus(cluster *fedv1b1.Kube
 		klog.Warningf("Failed to update the status of cluster %q: %v", cluster.Name, err)
 	}
 
-	metrics.UpdateDurationFromStart(metrics.ClusterHealthStatus, clusterHealthStatusStart)
+	metrics.ClusterHealthStatusDurationFromStart(clusterHealthStatusStart)
 
 	wg.Done()
 }

pkg/controller/sync/controller.go

@@ -267,7 +267,7 @@ func (s *KubeFedSyncController) reconcile(qualifiedName util.QualifiedName) util
 	startTime := time.Now()
 	defer func() {
 		klog.V(4).Infof("Finished reconciling %s %q (duration: %v)", kind, key, time.Since(startTime))
-		metrics.UpdateDurationFromStart(metrics.ReconcileFederatedResources, startTime)
+		metrics.ReconcileFederatedResourcesDurationFromStart(startTime)
 	}()
 
 	if fedResource.Object().GetDeletionTimestamp() != nil {

pkg/controller/sync/dispatch/managed.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/pkg/errors"
 
@@ -33,6 +34,7 @@ import (
 	"sigs.k8s.io/kubefed/pkg/client/generic"
 	"sigs.k8s.io/kubefed/pkg/controller/sync/status"
 	"sigs.k8s.io/kubefed/pkg/controller/util"
+	"sigs.k8s.io/kubefed/pkg/metrics"
 )
 
 // FederatedResourceForDispatch is the subset of the FederatedResource
@@ -130,7 +132,7 @@ func (d *managedDispatcherImpl) Create(clusterName string) {
 	// operation timed out.  The timeout status will be cleared by
 	// Wait() if a timeout does not occur.
 	d.RecordStatus(clusterName, status.CreationTimedOut)
-
+	start := time.Now()
 	d.dispatcher.incrementOperationsInitiated()
 	const op = "create"
 	go d.dispatcher.clusterOperation(clusterName, op, func(client generic.Client) util.ReconciliationStatus {
@@ -150,6 +152,7 @@ func (d *managedDispatcherImpl) Create(clusterName string) {
 		if err == nil {
 			version := util.ObjectVersion(obj)
 			d.recordVersion(clusterName, version)
+			metrics.DispatchOperationDurationFromStart("create", start)
 			return util.StatusAllOK
 		}
 
@@ -175,6 +178,7 @@ func (d *managedDispatcherImpl) Create(clusterName string) {
 
 		d.recordError(clusterName, op, errors.Errorf("An update will be attempted instead of a creation due to an existing resource"))
 		d.Update(clusterName, obj)
+		metrics.DispatchOperationDurationFromStart("update", start)
 		return util.StatusAllOK
 	})
 }

pkg/controller/sync/dispatch/unmanaged.go

@@ -18,6 +18,7 @@ package dispatch
 
 import (
 	"context"
+	"time"
 
 	"github.com/pkg/errors"
 
@@ -30,6 +31,7 @@ import (
 	"sigs.k8s.io/kubefed/pkg/client/generic"
 	"sigs.k8s.io/kubefed/pkg/controller/sync/status"
 	"sigs.k8s.io/kubefed/pkg/controller/util"
+	"sigs.k8s.io/kubefed/pkg/metrics"
 )
 
 const eventTemplate = "%s %s %q in cluster %q"
@@ -71,6 +73,7 @@ func (d *unmanagedDispatcherImpl) Wait() (bool, error) {
 }
 
 func (d *unmanagedDispatcherImpl) Delete(clusterName string) {
+	start := time.Now()
 	d.dispatcher.incrementOperationsInitiated()
 	const op = "delete"
 	const opContinuous = "Deleting"
@@ -97,6 +100,7 @@ func (d *unmanagedDispatcherImpl) Delete(clusterName string) {
 			}
 			return util.StatusError
 		}
+		metrics.DispatchOperationDurationFromStart("delete", start)
 		return util.StatusAllOK
 	})
 }

pkg/controller/util/federated_informer.go

@@ -341,7 +341,7 @@ func (f *federatedInformerImpl) GetClientForCluster(clusterName string) (generic
 	}
 	f.clusterClients[clusterName] = client
 
-	metrics.UpdateDurationFromStart(metrics.ClusterClientConnection, clientConnectionStart)
+	metrics.ClusterClientConnectionDurationFromStart(clientConnectionStart)
 	return client, nil
 }
 

pkg/kubefedctl/join.go

@@ -43,6 +43,7 @@ import (
 	ctlutil "sigs.k8s.io/kubefed/pkg/controller/util"
 	"sigs.k8s.io/kubefed/pkg/kubefedctl/options"
 	"sigs.k8s.io/kubefed/pkg/kubefedctl/util"
+	"sigs.k8s.io/kubefed/pkg/metrics"
 )
 
 const (
@@ -218,6 +219,7 @@ func JoinCluster(hostConfig, clusterConfig *rest.Config, kubefedNamespace,
 func joinClusterForNamespace(hostConfig, clusterConfig *rest.Config, kubefedNamespace,
 	joiningNamespace, hostClusterName, joiningClusterName, secretName string,
 	scope apiextv1b1.ResourceScope, dryRun, errorOnExisting bool) (*fedv1b1.KubeFedCluster, error) {
+	start := time.Now()
 
 	hostClientset, err := util.HostClientset(hostConfig)
 	if err != nil {
@@ -280,6 +282,8 @@ func joinClusterForNamespace(hostConfig, clusterConfig *rest.Config, kubefedName
 	}
 
 	klog.V(2).Info("Created federated cluster resource")
+	metrics.JoinedClusterTotalInc()
+	metrics.JoinedClusterDurationFromStart(start)
 	return kubefedCluster, nil
 }
 

pkg/kubefedctl/unjoin.go

@@ -21,6 +21,7 @@ import (
 	goerrors "errors"
 	"io"
 	"strings"
+	"time"
 
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
@@ -37,6 +38,7 @@ import (
 	controllerutil "sigs.k8s.io/kubefed/pkg/controller/util"
 	"sigs.k8s.io/kubefed/pkg/kubefedctl/options"
 	"sigs.k8s.io/kubefed/pkg/kubefedctl/util"
+	"sigs.k8s.io/kubefed/pkg/metrics"
 )
 
 var (
@@ -172,6 +174,7 @@ func (j *unjoinFederation) Run(cmdOut io.Writer, config util.FedConfig) error {
 // required set of parameters are passed in.
 func UnjoinCluster(hostConfig, clusterConfig *rest.Config, kubefedNamespace, hostClusterName,
 	unjoiningClusterContext, unjoiningClusterName string, forceDeletion, dryRun bool) error {
+	start := time.Now()
 
 	hostClientset, err := util.HostClientset(hostConfig)
 	if err != nil {
@@ -215,7 +218,13 @@ func UnjoinCluster(hostConfig, clusterConfig *rest.Config, kubefedNamespace, hos
 	}
 
 	// deletionSucceeded when all operations in deleteRBACResources and deleteFedNSFromUnjoinCluster succeed.
-	return deleteFederatedClusterAndSecret(hostClientset, client, kubefedNamespace, unjoiningClusterName, forceDeletion, dryRun)
+	err = deleteFederatedClusterAndSecret(hostClientset, client, kubefedNamespace, unjoiningClusterName, forceDeletion, dryRun)
+	if err != nil {
+		return err
+	}
+	metrics.JoinedClusterTotalDec()
+	metrics.UnjoinedClusterDurationFromStart(start)
+	return nil
 }
 
 // deleteKubeFedClusterAndSecret deletes a federated cluster resource that associates