gaugevec metrics for pkg repo, app, pkg install

cmd/controller/run.go

@@ -231,7 +231,8 @@ func Run(opts Options, runLog logr.Logger) error {
 		pkgToPkgInstallHandler := pkginstall.NewPackageInstallVersionHandler(
 			kcClient, opts.PackagingGlobalNS, runLog.WithName("handler"))
 
-		reconciler := pkginstall.NewReconciler(kcClient, pkgClient, coreClient, pkgToPkgInstallHandler, runLog.WithName("pkgi"), compInfo, kcConfig)
+		reconciler := pkginstall.NewReconciler(kcClient, pkgClient, coreClient, pkgToPkgInstallHandler,
+			runLog.WithName("pkgi"), compInfo, kcConfig, reconcileTimeMetrics)
 
 		ctrl, err := controller.New("pkgi", mgr, controller.Options{
 			Reconciler:              reconciler,
@@ -258,6 +259,7 @@ func Run(opts Options, runLog logr.Logger) error {
 			CoreClient:  coreClient,
 			AppClient:   kcClient,
 			KcConfig:    kcConfig,
+			AppMetrics:  appMetrics,
 			TimeMetrics: reconcileTimeMetrics,
 			CmdRunner:   sidecarCmdExec,
 			Kubeconf:    kubeconf,

config/config/agg-api.yml

@@ -23,5 +23,10 @@ spec:
     - port: 443
       protocol: TCP
       targetPort: api
+      name: main
+    - port: 8080
+      protocol: TCP
+      targetPort: metrics
+      name: metrics
   selector:
     app: kapp-controller

config/config/deployment.yml

@@ -52,6 +52,9 @@ spec:
         - containerPort: #@ data.values.apiPort
           name: api
           protocol: TCP
+        - containerPort: 8080
+          name: metrics
+          protocol: TCP
         securityContext:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true

go.mod

@@ -30,6 +30,7 @@ require (
 	github.com/cppforlife/go-cli-ui v0.0.0-20220425131040-94f26b16bc14
 	github.com/go-logr/logr v1.2.4
 	github.com/k14s/semver/v4 v4.0.1-0.20210701191048-266d47ac6115
+	github.com/prometheus/client_model v0.4.0
 	github.com/spf13/cobra v1.6.1
 	golang.org/x/sync v0.2.0
 	gopkg.in/yaml.v2 v2.4.0
@@ -86,7 +87,6 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.4.0 // indirect
 	github.com/prometheus/common v0.42.0 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect

pkg/app/app.go

@@ -60,6 +60,7 @@ type App struct {
 	appMetrics  *metrics.AppMetrics
 	timeMetrics *metrics.ReconcileTimeMetrics
 
+	isFirstReconcile      string
 	pendingStatusUpdate   bool
 	flushAllStatusUpdates bool
 	metadata              *deploy.Meta

pkg/app/app_deploy.go

@@ -5,20 +5,13 @@ package app
 
 import (
 	"fmt"
-	"time"
-
 	"github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/kappctrl/v1alpha1"
 	ctldep "github.com/vmware-tanzu/carvel-kapp-controller/pkg/deploy"
 	"github.com/vmware-tanzu/carvel-kapp-controller/pkg/exec"
 	"github.com/vmware-tanzu/carvel-kapp-controller/pkg/kubeconfig"
 )
 
 func (a *App) deploy(tplOutput string, changedFunc func(exec.CmdRunResult)) exec.CmdRunResult {
-	reconcileStartTS := time.Now()
-	defer func() {
-		a.timeMetrics.RegisterDeployTime(a.app.Kind, a.app.Name, a.app.Namespace, "", time.Since(reconcileStartTS))
-	}()
-
 	err := a.blockDeletion()
 	if err != nil {
 		return exec.NewCmdRunResultWithErr(fmt.Errorf("Blocking for deploy: %s", err))

pkg/app/app_reconcile.go

@@ -104,8 +104,14 @@ func (a *App) reconcileDeploy() error {
 
 func (a *App) reconcileFetchTemplateDeploy() exec.CmdRunResult {
 	reconcileStartTS := time.Now()
+	a.isFirstReconcile = "false"
+	if a.appMetrics.GetReconcileAttemptCounterValue(a.app.Name, a.app.Namespace) == 1 {
+		a.isFirstReconcile = "true"
+	}
+
 	defer func() {
-		a.timeMetrics.RegisterOverallTime(a.app.Kind, a.app.Name, a.app.Namespace, "", time.Since(reconcileStartTS))
+		a.timeMetrics.RegisterOverallTime(a.app.Kind, a.app.Name, a.app.Namespace, a.isFirstReconcile,
+			time.Since(reconcileStartTS))
 	}()
 
 	tmpDir := memdir.NewTmpDir("fetch-template-deploy")
@@ -134,6 +140,9 @@ func (a *App) reconcileFetchTemplateDeploy() exec.CmdRunResult {
 			UpdatedAt: metav1.NewTime(time.Now().UTC()),
 		}
 
+		a.timeMetrics.RegisterFetchTime(a.app.Kind, a.app.Name, a.app.Namespace, a.isFirstReconcile,
+			a.app.Status.Fetch.UpdatedAt.Sub(a.app.Status.Fetch.StartedAt.Time))
+
 		err := a.updateStatus("marking fetch completed")
 		if err != nil {
 			return exec.NewCmdRunResultWithErr(err)
@@ -144,6 +153,8 @@ func (a *App) reconcileFetchTemplateDeploy() exec.CmdRunResult {
 		}
 	}
 
+	templateStartTime := time.Now()
+
 	tplResult := a.template(assetsPath)
 
 	a.app.Status.Template = &v1alpha1.AppStatusTemplate{
@@ -153,6 +164,9 @@ func (a *App) reconcileFetchTemplateDeploy() exec.CmdRunResult {
 		UpdatedAt: metav1.NewTime(time.Now().UTC()),
 	}
 
+	a.timeMetrics.RegisterTemplateTime(a.app.Kind, a.app.Name, a.app.Namespace, a.isFirstReconcile,
+		a.app.Status.Template.UpdatedAt.Sub(templateStartTime))
+
 	err = a.updateStatus("marking template completed")
 	if err != nil {
 		return exec.NewCmdRunResultWithErr(err)
@@ -201,6 +215,9 @@ func (a *App) updateLastDeploy(result exec.CmdRunResult) exec.CmdRunResult {
 		},
 	}
 
+	a.timeMetrics.RegisterDeployTime(a.app.Kind, a.app.Name, a.app.Namespace, a.isFirstReconcile,
+		a.Status().Deploy.UpdatedAt.Sub(a.Status().Deploy.StartedAt.Time))
+
 	return result
 }
 

pkg/app/app_template.go

@@ -5,20 +5,13 @@ package app
 
 import (
 	"fmt"
-	"strings"
-	"time"
-
 	"github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/kappctrl/v1alpha1"
 	"github.com/vmware-tanzu/carvel-kapp-controller/pkg/exec"
 	ctltpl "github.com/vmware-tanzu/carvel-kapp-controller/pkg/template"
+	"strings"
 )
 
 func (a *App) template(dirPath string) exec.CmdRunResult {
-	reconcileStartTS := time.Now()
-	defer func() {
-		a.timeMetrics.RegisterTemplateTime(a.app.Kind, a.app.Name, a.app.Namespace, "", time.Since(reconcileStartTS))
-	}()
-
 	if len(a.app.Spec.Template) == 0 {
 		return exec.NewCmdRunResultWithErr(fmt.Errorf("Expected at least one template option"))
 	}

pkg/metrics/app_metrics.go

@@ -7,6 +7,7 @@ import (
 	"sync"
 
 	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
 )
 
@@ -131,3 +132,11 @@ func (am *AppMetrics) RegisterReconcileDeleteAttempt(appName string, namespace s
 func (am *AppMetrics) RegisterReconcileDeleteFailed(appName string, namespace string) {
 	am.reconcileDeleteFailedTotal.WithLabelValues(appName, namespace).Inc()
 }
+
+func (am *AppMetrics) GetReconcileAttemptCounterValue(appName, namespace string) int64 {
+	var m = &dto.Metric{}
+	if err := am.reconcileAttemptTotal.WithLabelValues(appName, namespace).Write(m); err != nil {
+		return 0
+	}
+	return int64(m.Counter.GetValue())
+}

pkg/metrics/reconcile_time_metrics.go

@@ -3,6 +3,7 @@ package metrics
 import (
 	"github.com/prometheus/client_golang/prometheus"
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
+	"sync"
 	"time"
 )
 
@@ -13,6 +14,10 @@ type ReconcileTimeMetrics struct {
 	reconcileTemplateTimeSeconds *prometheus.GaugeVec
 }
 
+var (
+	timeMetricsOnce sync.Once
+)
+
 func NewReconcileTimeMetrics() *ReconcileTimeMetrics {
 	const (
 		metricNamespace     = "kappctrl_reconcile_time_seconds"
@@ -58,7 +63,7 @@ func NewReconcileTimeMetrics() *ReconcileTimeMetrics {
 }
 
 func (tm *ReconcileTimeMetrics) RegisterAllMetrics() {
-	once.Do(func() {
+	timeMetricsOnce.Do(func() {
 		metrics.Registry.MustRegister(
 			tm.reconcileTimeSeconds,
 			tm.reconcileFetchTimeSeconds,

pkg/packageinstall/app.go

@@ -5,16 +5,15 @@ package packageinstall
 
 import (
 	"fmt"
-	"sort"
-	"strings"
-
 	"github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/kappctrl/v1alpha1"
 	kcv1alpha1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/kappctrl/v1alpha1"
 	pkgingv1alpha1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/packaging/v1alpha1"
 	datapkgingv1alpha1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apiserver/apis/datapackaging/v1alpha1"
 	"github.com/vmware-tanzu/carvel-kapp-controller/pkg/client/clientset/versioned/scheme"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sort"
+	"strings"
 )
 
 const (

pkg/packageinstall/packageinstall.go

@@ -6,6 +6,7 @@ package packageinstall
 import (
 	"context"
 	"fmt"
+	"github.com/vmware-tanzu/carvel-kapp-controller/pkg/metrics"
 	"time"
 
 	"github.com/go-logr/logr"
@@ -54,6 +55,8 @@ type PackageInstallCR struct {
 	coreClient kubernetes.Interface
 	compInfo   ComponentInfo
 	opts       Opts
+
+	timeMetrics *metrics.ReconcileTimeMetrics
 }
 
 // nolint: revive
@@ -62,10 +65,11 @@ type Opts struct {
 }
 
 func NewPackageInstallCR(model *pkgingv1alpha1.PackageInstall, log logr.Logger,
-	kcclient kcclient.Interface, pkgclient pkgclient.Interface, coreClient kubernetes.Interface, compInfo ComponentInfo, opts Opts) *PackageInstallCR {
+	kcclient kcclient.Interface, pkgclient pkgclient.Interface, coreClient kubernetes.Interface,
+	compInfo ComponentInfo, opts Opts, timeMetrics *metrics.ReconcileTimeMetrics) *PackageInstallCR {
 
 	return &PackageInstallCR{model: model, unmodifiedModel: model.DeepCopy(), log: log,
-		kcclient: kcclient, pkgclient: pkgclient, coreClient: coreClient, compInfo: compInfo, opts: opts}
+		kcclient: kcclient, pkgclient: pkgclient, coreClient: coreClient, compInfo: compInfo, opts: opts, timeMetrics: timeMetrics}
 }
 
 func (pi *PackageInstallCR) Reconcile() (reconcile.Result, error) {
@@ -101,6 +105,11 @@ func (pi *PackageInstallCR) Reconcile() (reconcile.Result, error) {
 func (pi *PackageInstallCR) reconcile(modelStatus *reconciler.Status) (reconcile.Result, error) {
 	pi.log.Info("Reconciling")
 
+	reconcileStartTS := time.Now()
+	defer func() {
+		pi.timeMetrics.RegisterOverallTime("pkgi", pi.model.Name, pi.model.Namespace, "", time.Since(reconcileStartTS))
+	}()
+
 	err := pi.blockDeletion()
 	if err != nil {
 		return reconcile.Result{Requeue: true}, err

pkg/packageinstall/packageinstall_deletion_test.go

@@ -4,6 +4,7 @@
 package packageinstall
 
 import (
+	"github.com/vmware-tanzu/carvel-kapp-controller/pkg/metrics"
 	"testing"
 
 	"github.com/k14s/semver/v4"
@@ -65,7 +66,7 @@ func Test_PackageInstallDeletion(t *testing.T) {
 		appClient := fakekappctrl.NewSimpleClientset(pkgInstall, existingApp)
 		coreClient := fake.NewSimpleClientset()
 
-		ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, coreClient, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{})
+		ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, coreClient, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}, &metrics.ReconcileTimeMetrics{})
 		_, err := ip.Reconcile()
 		assert.Nil(t, err)
 

pkg/packageinstall/packageinstall_downgrade_test.go

@@ -4,6 +4,7 @@
 package packageinstall
 
 import (
+	"github.com/vmware-tanzu/carvel-kapp-controller/pkg/metrics"
 	"testing"
 
 	"github.com/k14s/semver/v4"
@@ -25,6 +26,7 @@ import (
 
 func Test_PackageInstallVersionDowngrades(t *testing.T) {
 	log := logf.Log.WithName("kc")
+	var reconcileTimeMetrics = metrics.NewReconcileTimeMetrics()
 
 	pkg1 := datapkgingv1alpha1.Package{
 		ObjectMeta: metav1.ObjectMeta{
@@ -98,7 +100,7 @@ func Test_PackageInstallVersionDowngrades(t *testing.T) {
 			GitVersion: "v0.20.0",
 		}
 
-		ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{})
+		ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}, reconcileTimeMetrics)
 		_, err := ip.Reconcile()
 		assert.Nil(t, err)
 
@@ -147,7 +149,7 @@ func Test_PackageInstallVersionDowngrades(t *testing.T) {
 			GitVersion: "v0.20.0",
 		}
 
-		ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{})
+		ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}, reconcileTimeMetrics)
 		_, err := ip.Reconcile()
 		assert.Nil(t, err)
 
@@ -196,7 +198,7 @@ func Test_PackageInstallVersionDowngrades(t *testing.T) {
 			GitVersion: "v0.20.0",
 		}
 
-		ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{})
+		ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}, reconcileTimeMetrics)
 		_, err := ip.Reconcile()
 		assert.Nil(t, err)
 
@@ -251,7 +253,7 @@ func Test_PackageInstallVersionDowngrades(t *testing.T) {
 			GitVersion: "v0.20.0",
 		}
 
-		ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{})
+		ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}, reconcileTimeMetrics)
 		_, err := ip.Reconcile()
 		assert.Nil(t, err)