diff --git a/cmd/controller/run.go b/cmd/controller/run.go index 7d7826925..223c36f8b 100644 --- a/cmd/controller/run.go +++ b/cmd/controller/run.go @@ -95,10 +95,6 @@ func Run(opts Options, runLog logr.Logger) error { return fmt.Errorf("Building packaging client: %s", err) } - runLog.Info("setting up metrics") - appMetrics := metrics.NewAppMetrics() - appMetrics.RegisterAllMetrics() - var server *apiserver.APIServer if opts.StartAPIServer { // assign bindPort to env var KAPPCTRL_API_PORT if available @@ -187,6 +183,11 @@ func Run(opts Options, runLog logr.Logger) error { kubeconf := kubeconfig.NewKubeconfig(coreClient, runLog) compInfo := componentinfo.NewComponentInfo(coreClient, kubeconf, Version) + runLog.Info("setting up metrics") + appMetrics := metrics.NewMetrics() + appMetrics.ReconcileTimeMetrics.RegisterAllMetrics() + appMetrics.ReconcileCountMetrics.RegisterAllMetrics() + cacheFolderApps := memdir.NewTmpDir("cache-appcr") err = cacheFolderApps.Create() if err != nil { @@ -227,7 +228,8 @@ func Run(opts Options, runLog logr.Logger) error { pkgToPkgInstallHandler := pkginstall.NewPackageInstallVersionHandler( kcClient, opts.PackagingGlobalNS, runLog.WithName("handler")) - reconciler := pkginstall.NewReconciler(kcClient, pkgClient, coreClient, pkgToPkgInstallHandler, runLog.WithName("pkgi"), compInfo, kcConfig) + reconciler := pkginstall.NewReconciler(kcClient, pkgClient, coreClient, pkgToPkgInstallHandler, + runLog.WithName("pkgi"), compInfo, kcConfig, appMetrics) ctrl, err := controller.New("pkgi", mgr, controller.Options{ Reconciler: reconciler, @@ -254,6 +256,7 @@ func Run(opts Options, runLog logr.Logger) error { CoreClient: coreClient, AppClient: kcClient, KcConfig: kcConfig, + AppMetrics: appMetrics, CmdRunner: sidecarCmdExec, Kubeconf: kubeconf, CacheFolder: cacheFolderPkgRepoApps, diff --git a/config/config/agg-api.yml b/config/config/agg-api.yml index 5a0912822..181b9708c 100644 --- a/config/config/agg-api.yml +++ b/config/config/agg-api.yml @@ -23,5 +23,10 @@ spec: - port: 443 protocol: TCP targetPort: api + name: main + - port: 8080 + protocol: TCP + targetPort: metrics + name: metrics selector: app: kapp-controller diff --git a/config/config/deployment.yml b/config/config/deployment.yml index 11b138032..2aabfe770 100644 --- a/config/config/deployment.yml +++ b/config/config/deployment.yml @@ -52,6 +52,9 @@ spec: - containerPort: #@ data.values.apiPort name: api protocol: TCP + - containerPort: #@ data.values.metricsPort + name: metrics + protocol: TCP securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true diff --git a/config/values-schema.yml b/config/values-schema.yml index 1504d9be7..fecbd8469 100644 --- a/config/values-schema.yml +++ b/config/values-schema.yml @@ -17,6 +17,8 @@ dangerousEnablePprof: false tlsCipherSuites: "" #@schema/desc "API port" apiPort: 8443 +#@schema/desc "Metrics port" +metricsPort: 8080 #@schema/desc "The coreDNSIP will be injected into /etc/resolv.conf of kapp-controller pod" coreDNSIP: "" #@schema/desc "HostNetwork of kapp-controller deployment." diff --git a/go.mod b/go.mod index 677508826..00f3d49eb 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,7 @@ require ( github.com/cppforlife/go-cli-ui v0.0.0-20220425131040-94f26b16bc14 github.com/go-logr/logr v1.2.4 github.com/k14s/semver/v4 v4.0.1-0.20210701191048-266d47ac6115 + github.com/prometheus/client_model v0.4.0 github.com/spf13/cobra v1.6.1 golang.org/x/sync v0.2.0 gopkg.in/yaml.v2 v2.4.0 @@ -86,7 +87,6 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/spf13/pflag v1.0.5 // indirect diff --git a/pkg/app/app.go b/pkg/app/app.go index 42027dcf9..e1e937074 100644 --- a/pkg/app/app.go +++ b/pkg/app/app.go @@ -57,7 +57,7 @@ type App struct { log logr.Logger opts Opts - appMetrics *metrics.AppMetrics + appMetrics *metrics.Metrics pendingStatusUpdate bool flushAllStatusUpdates bool @@ -66,7 +66,7 @@ type App struct { func NewApp(app v1alpha1.App, hooks Hooks, fetchFactory fetch.Factory, templateFactory template.Factory, - deployFactory deploy.Factory, log logr.Logger, opts Opts, appMetrics *metrics.AppMetrics, compInfo ComponentInfo) *App { + deployFactory deploy.Factory, log logr.Logger, opts Opts, appMetrics *metrics.Metrics, compInfo ComponentInfo) *App { return &App{app: app, appPrev: *(app.DeepCopy()), hooks: hooks, fetchFactory: fetchFactory, templateFactory: templateFactory, @@ -76,6 +76,9 @@ func NewApp(app v1alpha1.App, hooks Hooks, func (a *App) Name() string { return a.app.Name } func (a *App) Namespace() string { return a.app.Namespace } +// Kind return kind of App +func (a *App) Kind() string { return "App" } + func (a *App) Status() v1alpha1.AppStatus { return a.app.Status } func (a *App) StatusAsYAMLBytes() ([]byte, error) { diff --git a/pkg/app/app_factory.go b/pkg/app/app_factory.go index 4b42fb86d..79c290a27 100644 --- a/pkg/app/app_factory.go +++ b/pkg/app/app_factory.go @@ -26,7 +26,7 @@ type CRDAppFactory struct { CoreClient kubernetes.Interface AppClient kcclient.Interface KcConfig *config.Config - AppMetrics *metrics.AppMetrics + AppMetrics *metrics.Metrics VendirConfigHook func(vendirconf.Config) vendirconf.Config KbldAllowBuild bool CmdRunner exec.CmdRunner diff --git a/pkg/app/app_reconcile.go b/pkg/app/app_reconcile.go index fc36ab365..d4c615a65 100644 --- a/pkg/app/app_reconcile.go +++ b/pkg/app/app_reconcile.go @@ -21,7 +21,7 @@ func (a *App) Reconcile(force bool) (reconcile.Result, error) { var err error - a.appMetrics.InitMetrics(a.Name(), a.Namespace()) + a.appMetrics.ReconcileCountMetrics.InitMetrics(a.Kind(), a.Name(), a.Namespace()) timerOpts := ReconcileTimerOpts{ DefaultSyncPeriod: a.opts.DefaultSyncPeriod, @@ -103,6 +103,13 @@ func (a *App) reconcileDeploy() error { } func (a *App) reconcileFetchTemplateDeploy() exec.CmdRunResult { + reconcileStartTime := time.Now() + a.appMetrics.IsFirstReconcile = a.appMetrics.ReconcileCountMetrics.GetReconcileAttemptCounterValue(a.Kind(), a.Name(), a.Namespace()) == 1 + defer func() { + a.appMetrics.ReconcileTimeMetrics.RegisterOverallTime(a.Kind(), a.Name(), a.Namespace(), a.appMetrics.IsFirstReconcile, + time.Since(reconcileStartTime)) + }() + tmpDir := memdir.NewTmpDir("fetch-template-deploy") err := tmpDir.Create() @@ -129,6 +136,9 @@ func (a *App) reconcileFetchTemplateDeploy() exec.CmdRunResult { UpdatedAt: metav1.NewTime(time.Now().UTC()), } + a.appMetrics.ReconcileTimeMetrics.RegisterFetchTime(a.Kind(), a.Name(), a.Namespace(), a.appMetrics.IsFirstReconcile, + a.app.Status.Fetch.UpdatedAt.Sub(a.app.Status.Fetch.StartedAt.Time)) + err := a.updateStatus("marking fetch completed") if err != nil { return exec.NewCmdRunResultWithErr(err) @@ -139,6 +149,8 @@ func (a *App) reconcileFetchTemplateDeploy() exec.CmdRunResult { } } + templateStartTime := time.Now() + tplResult := a.template(assetsPath) a.app.Status.Template = &v1alpha1.AppStatusTemplate{ @@ -148,6 +160,9 @@ func (a *App) reconcileFetchTemplateDeploy() exec.CmdRunResult { UpdatedAt: metav1.NewTime(time.Now().UTC()), } + a.appMetrics.ReconcileTimeMetrics.RegisterTemplateTime(a.Kind(), a.Name(), a.Namespace(), a.appMetrics.IsFirstReconcile, + a.app.Status.Template.UpdatedAt.Sub(templateStartTime)) + err = a.updateStatus("marking template completed") if err != nil { return exec.NewCmdRunResultWithErr(err) @@ -196,6 +211,9 @@ func (a *App) updateLastDeploy(result exec.CmdRunResult) exec.CmdRunResult { }, } + a.appMetrics.ReconcileTimeMetrics.RegisterDeployTime(a.Kind(), a.Name(), a.Namespace(), a.appMetrics.IsFirstReconcile, + a.Status().Deploy.UpdatedAt.Sub(a.Status().Deploy.StartedAt.Time)) + return result } @@ -247,7 +265,7 @@ func (a *App) setReconciling() { Status: corev1.ConditionTrue, }) - a.appMetrics.RegisterReconcileAttempt(a.app.Name, a.app.Namespace) + a.appMetrics.ReconcileCountMetrics.RegisterReconcileAttempt(a.Kind(), a.Name(), a.Namespace()) a.app.Status.FriendlyDescription = "Reconciling" } @@ -263,7 +281,7 @@ func (a *App) setReconcileCompleted(result exec.CmdRunResult) { a.app.Status.ConsecutiveReconcileFailures++ a.app.Status.ConsecutiveReconcileSuccesses = 0 a.app.Status.FriendlyDescription = fmt.Sprintf("Reconcile failed: %s", result.ErrorStr()) - a.appMetrics.RegisterReconcileFailure(a.app.Name, a.app.Namespace) + a.appMetrics.ReconcileCountMetrics.RegisterReconcileFailure(a.Kind(), a.Name(), a.Namespace()) a.setUsefulErrorMessage(result) } else { a.app.Status.Conditions = append(a.app.Status.Conditions, v1alpha1.Condition{ @@ -274,7 +292,7 @@ func (a *App) setReconcileCompleted(result exec.CmdRunResult) { a.app.Status.ConsecutiveReconcileSuccesses++ a.app.Status.ConsecutiveReconcileFailures = 0 a.app.Status.FriendlyDescription = "Reconcile succeeded" - a.appMetrics.RegisterReconcileSuccess(a.app.Name, a.app.Namespace) + a.appMetrics.ReconcileCountMetrics.RegisterReconcileSuccess(a.Kind(), a.Name(), a.Namespace()) a.app.Status.UsefulErrorMessage = "" } } @@ -287,7 +305,7 @@ func (a *App) setDeleting() { Status: corev1.ConditionTrue, }) - a.appMetrics.RegisterReconcileDeleteAttempt(a.app.Name, a.app.Namespace) + a.appMetrics.ReconcileCountMetrics.RegisterReconcileDeleteAttempt(a.Kind(), a.Name(), a.Namespace()) a.app.Status.FriendlyDescription = "Deleting" } @@ -303,10 +321,10 @@ func (a *App) setDeleteCompleted(result exec.CmdRunResult) { a.app.Status.ConsecutiveReconcileFailures++ a.app.Status.ConsecutiveReconcileSuccesses = 0 a.app.Status.FriendlyDescription = fmt.Sprintf("Delete failed: %s", result.ErrorStr()) - a.appMetrics.RegisterReconcileDeleteFailed(a.app.Name, a.app.Namespace) + a.appMetrics.ReconcileCountMetrics.RegisterReconcileDeleteFailed(a.Kind(), a.Name(), a.Namespace()) a.setUsefulErrorMessage(result) } else { - a.appMetrics.DeleteMetrics(a.app.Name, a.app.Namespace) + a.appMetrics.ReconcileCountMetrics.DeleteMetrics(a.Kind(), a.Name(), a.Namespace()) } } diff --git a/pkg/app/app_reconcile_test.go b/pkg/app/app_reconcile_test.go index 47755dd70..e5cbab246 100644 --- a/pkg/app/app_reconcile_test.go +++ b/pkg/app/app_reconcile_test.go @@ -29,7 +29,6 @@ import ( func Test_NoInspectReconcile_IfNoDeployAttempted(t *testing.T) { log := logf.Log.WithName("kc") - var appMetrics = metrics.NewAppMetrics() // The url under fetch is invalid, which will cause this // app to fail before deploy. @@ -52,7 +51,7 @@ func Test_NoInspectReconcile_IfNoDeployAttempted(t *testing.T) { tmpFac := template.NewFactory(k8scs, fetchFac, false, exec.NewPlainCmdRunner()) deployFac := deploy.NewFactory(k8scs, kubeconfig.NewKubeconfig(k8scs, log), nil, exec.NewPlainCmdRunner(), log) - crdApp := NewCRDApp(&app, log, appMetrics, kappcs, fetchFac, tmpFac, deployFac, FakeComponentInfo{}, Opts{MinimumSyncPeriod: 30 * time.Second}) + crdApp := NewCRDApp(&app, log, metrics.NewMetrics(), kappcs, fetchFac, tmpFac, deployFac, FakeComponentInfo{}, Opts{MinimumSyncPeriod: 30 * time.Second}) _, err := crdApp.Reconcile(false) assert.Nil(t, err, "unexpected error with reconciling", err) @@ -86,7 +85,6 @@ func Test_NoInspectReconcile_IfNoDeployAttempted(t *testing.T) { func Test_NoInspectReconcile_IfInspectNotEnabled(t *testing.T) { log := logf.Log.WithName("kc") - var appMetrics = metrics.NewAppMetrics() app := v1alpha1.App{ ObjectMeta: metav1.ObjectMeta{ @@ -119,7 +117,7 @@ func Test_NoInspectReconcile_IfInspectNotEnabled(t *testing.T) { tmpFac := template.NewFactory(k8scs, fetchFac, false, exec.NewPlainCmdRunner()) deployFac := deploy.NewFactory(k8scs, kubeconfig.NewKubeconfig(k8scs, log), nil, exec.NewPlainCmdRunner(), log) - crdApp := NewCRDApp(&app, log, appMetrics, kappcs, fetchFac, tmpFac, deployFac, FakeComponentInfo{}, Opts{MinimumSyncPeriod: 30 * time.Second}) + crdApp := NewCRDApp(&app, log, metrics.NewMetrics(), kappcs, fetchFac, tmpFac, deployFac, FakeComponentInfo{}, Opts{MinimumSyncPeriod: 30 * time.Second}) _, err := crdApp.Reconcile(false) assert.Nil(t, err, "unexpected error with reconciling", err) @@ -164,7 +162,6 @@ func Test_NoInspectReconcile_IfInspectNotEnabled(t *testing.T) { func Test_TemplateError_DisplayedInStatus_UsefulErrorMessageProperty(t *testing.T) { log := logf.Log.WithName("kc") - var appMetrics = metrics.NewAppMetrics() fetchInline := map[string]string{ "file.yml": `foo: #@ data.values.nothere`, @@ -191,7 +188,7 @@ func Test_TemplateError_DisplayedInStatus_UsefulErrorMessageProperty(t *testing. tmpFac := template.NewFactory(k8scs, fetchFac, false, exec.NewPlainCmdRunner()) deployFac := deploy.NewFactory(k8scs, kubeconfig.NewKubeconfig(k8scs, log), nil, exec.NewPlainCmdRunner(), log) - crdApp := NewCRDApp(&app, log, appMetrics, kappcs, fetchFac, tmpFac, deployFac, FakeComponentInfo{}, Opts{MinimumSyncPeriod: 30 * time.Second}) + crdApp := NewCRDApp(&app, log, metrics.NewMetrics(), kappcs, fetchFac, tmpFac, deployFac, FakeComponentInfo{}, Opts{MinimumSyncPeriod: 30 * time.Second}) _, err := crdApp.Reconcile(false) assert.Nil(t, err, "Unexpected error with reconciling", err) diff --git a/pkg/app/app_template_test.go b/pkg/app/app_template_test.go index f66d89c21..4e3941d31 100644 --- a/pkg/app/app_template_test.go +++ b/pkg/app/app_template_test.go @@ -61,7 +61,7 @@ func Test_BuildAdditionalDownwardAPIValues_MemoizedCallCount(t *testing.T) { K8sAPIsCount: &k8sAPIsCallCount, KCVersionCount: &kcVersionCallCount, } - app := NewApp(appEmpty, Hooks{}, fetchFac, tmpFac, deployFac, log, Opts{}, metrics.NewAppMetrics(), fakeInfo) + app := NewApp(appEmpty, Hooks{}, fetchFac, tmpFac, deployFac, log, Opts{}, metrics.NewMetrics(), fakeInfo) dir, err := os.MkdirTemp("", "temp") assert.NoError(t, err) diff --git a/pkg/app/app_test.go b/pkg/app/app_test.go index fc612db86..522147f44 100644 --- a/pkg/app/app_test.go +++ b/pkg/app/app_test.go @@ -15,6 +15,7 @@ import ( "github.com/vmware-tanzu/carvel-kapp-controller/pkg/exec" "github.com/vmware-tanzu/carvel-kapp-controller/pkg/fetch" "github.com/vmware-tanzu/carvel-kapp-controller/pkg/kubeconfig" + "github.com/vmware-tanzu/carvel-kapp-controller/pkg/metrics" "github.com/vmware-tanzu/carvel-kapp-controller/pkg/reftracker" "github.com/vmware-tanzu/carvel-kapp-controller/pkg/template" v1 "k8s.io/api/core/v1" @@ -64,7 +65,7 @@ func Test_SecretRefs_RetrievesAllSecretRefs(t *testing.T) { tmpFac := template.NewFactory(k8scs, fetchFac, false, exec.NewPlainCmdRunner()) deployFac := deploy.NewFactory(k8scs, kubeconfig.NewKubeconfig(k8scs, log), nil, exec.NewPlainCmdRunner(), log) - app := apppkg.NewApp(appWithRefs, apppkg.Hooks{}, fetchFac, tmpFac, deployFac, log, apppkg.Opts{}, nil, FakeComponentInfo{}) + app := apppkg.NewApp(appWithRefs, apppkg.Hooks{}, fetchFac, tmpFac, deployFac, log, apppkg.Opts{}, metrics.NewMetrics(), FakeComponentInfo{}) out := app.SecretRefs() assert.Truef(t, reflect.DeepEqual(out, expected), "Expected: %s\nGot: %s\n", expected, out) @@ -88,7 +89,7 @@ func Test_SecretRefs_RetrievesNoSecretRefs_WhenNonePresent(t *testing.T) { tmpFac := template.NewFactory(k8scs, fetchFac, false, exec.NewPlainCmdRunner()) deployFac := deploy.NewFactory(k8scs, kubeconfig.NewKubeconfig(k8scs, log), nil, exec.NewPlainCmdRunner(), log) - app := apppkg.NewApp(appEmpty, apppkg.Hooks{}, fetchFac, tmpFac, deployFac, log, apppkg.Opts{}, nil, FakeComponentInfo{}) + app := apppkg.NewApp(appEmpty, apppkg.Hooks{}, fetchFac, tmpFac, deployFac, log, apppkg.Opts{}, metrics.NewMetrics(), FakeComponentInfo{}) out := app.SecretRefs() assert.Equal(t, 0, len(out), "No SecretRefs to be returned") @@ -126,7 +127,7 @@ func Test_ConfigMapRefs_RetrievesAllConfigMapRefs(t *testing.T) { tmpFac := template.NewFactory(k8scs, fetchFac, false, exec.NewPlainCmdRunner()) deployFac := deploy.NewFactory(k8scs, kubeconfig.NewKubeconfig(k8scs, log), nil, exec.NewPlainCmdRunner(), log) - app := apppkg.NewApp(appWithRefs, apppkg.Hooks{}, fetchFac, tmpFac, deployFac, log, apppkg.Opts{}, nil, FakeComponentInfo{}) + app := apppkg.NewApp(appWithRefs, apppkg.Hooks{}, fetchFac, tmpFac, deployFac, log, apppkg.Opts{}, metrics.NewMetrics(), FakeComponentInfo{}) out := app.ConfigMapRefs() assert.Truef(t, reflect.DeepEqual(out, expected), "Expected: %s\nGot: %s\n", expected, out) @@ -150,7 +151,7 @@ func Test_ConfigMapRefs_RetrievesNoConfigMapRefs_WhenNonePresent(t *testing.T) { tmpFac := template.NewFactory(k8scs, fetchFac, false, exec.NewPlainCmdRunner()) deployFac := deploy.NewFactory(k8scs, kubeconfig.NewKubeconfig(k8scs, log), nil, exec.NewPlainCmdRunner(), log) - app := apppkg.NewApp(appEmpty, apppkg.Hooks{}, fetchFac, tmpFac, deployFac, log, apppkg.Opts{}, nil, FakeComponentInfo{}) + app := apppkg.NewApp(appEmpty, apppkg.Hooks{}, fetchFac, tmpFac, deployFac, log, apppkg.Opts{}, metrics.NewMetrics(), FakeComponentInfo{}) out := app.ConfigMapRefs() assert.Lenf(t, out, 0, "Expected: %s\nGot: %s\n", "No ConfigMapRefs to be returned", out) diff --git a/pkg/app/crd_app.go b/pkg/app/crd_app.go index 97132754b..0b3fcbe7d 100644 --- a/pkg/app/crd_app.go +++ b/pkg/app/crd_app.go @@ -21,20 +21,18 @@ import ( ) type CRDApp struct { - app *App - appModel *kcv1alpha1.App - log logr.Logger - appMetrics *metrics.AppMetrics - appClient kcclient.Interface + app *App + appModel *kcv1alpha1.App + log logr.Logger + appClient kcclient.Interface } // NewCRDApp creates new CRD app -func NewCRDApp(appModel *kcv1alpha1.App, log logr.Logger, appMetrics *metrics.AppMetrics, - appClient kcclient.Interface, fetchFactory fetch.Factory, +func NewCRDApp(appModel *kcv1alpha1.App, log logr.Logger, appMetrics *metrics.Metrics, appClient kcclient.Interface, fetchFactory fetch.Factory, templateFactory template.Factory, deployFactory deploy.Factory, compInfo ComponentInfo, opts Opts) *CRDApp { - crdApp := &CRDApp{appModel: appModel, log: log, appMetrics: appMetrics, appClient: appClient} + crdApp := &CRDApp{appModel: appModel, log: log, appClient: appClient} crdApp.app = NewApp(*appModel, Hooks{ BlockDeletion: crdApp.blockDeletion, diff --git a/pkg/metrics/app_metrics.go b/pkg/metrics/app_metrics.go deleted file mode 100644 index 434bae711..000000000 --- a/pkg/metrics/app_metrics.go +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright 2021 VMware, Inc. -// SPDX-License-Identifier: Apache-2.0 - -package metrics - -import ( - "sync" - - "github.com/prometheus/client_golang/prometheus" - "sigs.k8s.io/controller-runtime/pkg/metrics" -) - -// AppMetrics holds server metrics -type AppMetrics struct { - reconcileAttemptTotal *prometheus.CounterVec - reconcileSuccessTotal *prometheus.CounterVec - reconcileFailureTotal *prometheus.CounterVec - reconcileDeleteAttemptTotal *prometheus.CounterVec - reconcileDeleteFailedTotal *prometheus.CounterVec -} - -var ( - once sync.Once -) - -// NewAppMetrics creates AppMetrics object -func NewAppMetrics() *AppMetrics { - const ( - metricNamespace = "kappctrl" - kappNameLabel = "app_name" - kappNamespaceLabel = "namespace" - ) - return &AppMetrics{ - reconcileAttemptTotal: prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: metricNamespace, - Name: "app_reconcile_attempt_total", - Help: "Total number of attempted reconciles", - }, - []string{kappNameLabel, kappNamespaceLabel}, - ), - reconcileSuccessTotal: prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: metricNamespace, - Name: "app_reconcile_success_total", - Help: "Total number of succeeded reconciles", - }, - []string{kappNameLabel, kappNamespaceLabel}, - ), - reconcileFailureTotal: prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: metricNamespace, - Name: "app_reconcile_failure_total", - Help: "Total number of failed reconciles", - }, - []string{kappNameLabel, kappNamespaceLabel}, - ), - reconcileDeleteAttemptTotal: prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: metricNamespace, - Name: "app_reconcile_delete_attempt_total", - Help: "Total number of attempted reconcile deletion", - }, - []string{kappNameLabel, kappNamespaceLabel}, - ), - reconcileDeleteFailedTotal: prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: metricNamespace, - Name: "app_reconcile_delete_failed_total", - Help: "Total number of failed reconcile deletion", - }, - []string{kappNameLabel, kappNamespaceLabel}, - ), - } -} - -// RegisterAllMetrics registers all prometheus metrics. -func (am *AppMetrics) RegisterAllMetrics() { - once.Do(func() { - metrics.Registry.MustRegister( - am.reconcileAttemptTotal, - am.reconcileSuccessTotal, - am.reconcileFailureTotal, - am.reconcileDeleteAttemptTotal, - am.reconcileDeleteFailedTotal, - ) - }) -} - -// InitMetrics initializes metrics -func (am *AppMetrics) InitMetrics(appName string, namespace string) { - // Initializes counter metrics - am.reconcileAttemptTotal.WithLabelValues(appName, namespace).Add(0) - am.reconcileSuccessTotal.WithLabelValues(appName, namespace).Add(0) - am.reconcileFailureTotal.WithLabelValues(appName, namespace).Add(0) - am.reconcileDeleteAttemptTotal.WithLabelValues(appName, namespace).Add(0) - am.reconcileDeleteFailedTotal.WithLabelValues(appName, namespace).Add(0) -} - -// DeleteMetrics deletes metrics -func (am *AppMetrics) DeleteMetrics(appName string, namespace string) { - // Delete counter metrics - am.reconcileAttemptTotal.DeleteLabelValues(appName, namespace) - am.reconcileSuccessTotal.DeleteLabelValues(appName, namespace) - am.reconcileFailureTotal.DeleteLabelValues(appName, namespace) - am.reconcileDeleteAttemptTotal.DeleteLabelValues(appName, namespace) - am.reconcileDeleteFailedTotal.DeleteLabelValues(appName, namespace) -} - -// RegisterReconcileAttempt increments reconcileAttemptTotal -func (am *AppMetrics) RegisterReconcileAttempt(appName string, namespace string) { - am.reconcileAttemptTotal.WithLabelValues(appName, namespace).Inc() -} - -// RegisterReconcileSuccess increments reconcileSuccessTotal -func (am *AppMetrics) RegisterReconcileSuccess(appName string, namespace string) { - am.reconcileSuccessTotal.WithLabelValues(appName, namespace).Inc() -} - -// RegisterReconcileFailure increments reconcileFailureTotal -func (am *AppMetrics) RegisterReconcileFailure(appName string, namespace string) { - am.reconcileFailureTotal.WithLabelValues(appName, namespace).Inc() -} - -// RegisterReconcileDeleteAttempt increments reconcileDeleteAttemptTotal -func (am *AppMetrics) RegisterReconcileDeleteAttempt(appName string, namespace string) { - am.reconcileDeleteAttemptTotal.WithLabelValues(appName, namespace).Inc() -} - -// RegisterReconcileDeleteFailed increments reconcileDeleteFailedTotal -func (am *AppMetrics) RegisterReconcileDeleteFailed(appName string, namespace string) { - am.reconcileDeleteFailedTotal.WithLabelValues(appName, namespace).Inc() -} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go new file mode 100644 index 000000000..df69bd1f3 --- /dev/null +++ b/pkg/metrics/metrics.go @@ -0,0 +1,20 @@ +// Copyright 2021 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package metrics + +// Metrics holds all metrics +type Metrics struct { + *ReconcileCountMetrics + *ReconcileTimeMetrics + IsFirstReconcile bool +} + +// NewMetrics is a factory function that returns a new instance of Metrics. +func NewMetrics() *Metrics { + return &Metrics{ + ReconcileCountMetrics: NewCountMetrics(), + ReconcileTimeMetrics: NewReconcileTimeMetrics(), + IsFirstReconcile: false, + } +} diff --git a/pkg/metrics/reconcile_count_metrics.go b/pkg/metrics/reconcile_count_metrics.go new file mode 100644 index 000000000..27b14fc70 --- /dev/null +++ b/pkg/metrics/reconcile_count_metrics.go @@ -0,0 +1,144 @@ +// Copyright 2021 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package metrics + +import ( + "sync" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +// ReconcileCountMetrics holds server metrics +type ReconcileCountMetrics struct { + reconcileAttemptTotal *prometheus.CounterVec + reconcileSuccessTotal *prometheus.CounterVec + reconcileFailureTotal *prometheus.CounterVec + reconcileDeleteAttemptTotal *prometheus.CounterVec + reconcileDeleteFailedTotal *prometheus.CounterVec +} + +var ( + once sync.Once +) + +// NewCountMetrics creates ReconcileCountMetrics object +func NewCountMetrics() *ReconcileCountMetrics { + const ( + metricNamespace = "kappctrl" + kappNameLabel = "name" + kappNamespaceLabel = "namespace" + resourceTypeLabel = "controller" + ) + return &ReconcileCountMetrics{ + reconcileAttemptTotal: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricNamespace, + Name: "app_reconcile_attempt_total", + Help: "Total number of attempted reconciles", + }, + []string{resourceTypeLabel, kappNameLabel, kappNamespaceLabel}, + ), + reconcileSuccessTotal: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricNamespace, + Name: "app_reconcile_success_total", + Help: "Total number of succeeded reconciles", + }, + []string{resourceTypeLabel, kappNameLabel, kappNamespaceLabel}, + ), + reconcileFailureTotal: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricNamespace, + Name: "app_reconcile_failure_total", + Help: "Total number of failed reconciles", + }, + []string{resourceTypeLabel, kappNameLabel, kappNamespaceLabel}, + ), + reconcileDeleteAttemptTotal: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricNamespace, + Name: "app_reconcile_delete_attempt_total", + Help: "Total number of attempted reconcile deletions", + }, + []string{resourceTypeLabel, kappNameLabel, kappNamespaceLabel}, + ), + reconcileDeleteFailedTotal: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricNamespace, + Name: "app_reconcile_delete_failed_total", + Help: "Total number of failed reconcile deletions", + }, + []string{resourceTypeLabel, kappNameLabel, kappNamespaceLabel}, + ), + } +} + +// RegisterAllMetrics registers all prometheus metrics. +func (am *ReconcileCountMetrics) RegisterAllMetrics() { + once.Do(func() { + metrics.Registry.MustRegister( + am.reconcileAttemptTotal, + am.reconcileSuccessTotal, + am.reconcileFailureTotal, + am.reconcileDeleteAttemptTotal, + am.reconcileDeleteFailedTotal, + ) + }) +} + +// InitMetrics initializes metrics +func (am *ReconcileCountMetrics) InitMetrics(resourceType, name, namespace string) { + // Initializes counter metrics + am.reconcileAttemptTotal.WithLabelValues(resourceType, name, namespace).Add(0) + am.reconcileSuccessTotal.WithLabelValues(resourceType, name, namespace).Add(0) + am.reconcileFailureTotal.WithLabelValues(resourceType, name, namespace).Add(0) + am.reconcileDeleteAttemptTotal.WithLabelValues(resourceType, name, namespace).Add(0) + am.reconcileDeleteFailedTotal.WithLabelValues(resourceType, name, namespace).Add(0) +} + +// DeleteMetrics deletes metrics +func (am *ReconcileCountMetrics) DeleteMetrics(resourceType, name, namespace string) { + // Delete counter metrics + am.reconcileAttemptTotal.DeleteLabelValues(resourceType, name, namespace) + am.reconcileSuccessTotal.DeleteLabelValues(resourceType, name, namespace) + am.reconcileFailureTotal.DeleteLabelValues(resourceType, name, namespace) + am.reconcileDeleteAttemptTotal.DeleteLabelValues(resourceType, name, namespace) + am.reconcileDeleteFailedTotal.DeleteLabelValues(resourceType, name, namespace) +} + +// RegisterReconcileAttempt increments reconcileAttemptTotal +func (am *ReconcileCountMetrics) RegisterReconcileAttempt(resourceType, appName, namespace string) { + am.reconcileAttemptTotal.WithLabelValues(resourceType, appName, namespace).Inc() +} + +// RegisterReconcileSuccess increments reconcileSuccessTotal +func (am *ReconcileCountMetrics) RegisterReconcileSuccess(resourceType, appName, namespace string) { + am.reconcileSuccessTotal.WithLabelValues(resourceType, appName, namespace).Inc() +} + +// RegisterReconcileFailure increments reconcileFailureTotal +func (am *ReconcileCountMetrics) RegisterReconcileFailure(resourceType, appName, namespace string) { + am.reconcileFailureTotal.WithLabelValues(resourceType, appName, namespace).Inc() +} + +// RegisterReconcileDeleteAttempt increments reconcileDeleteAttemptTotal +func (am *ReconcileCountMetrics) RegisterReconcileDeleteAttempt(resourceType, appName, namespace string) { + am.reconcileDeleteAttemptTotal.WithLabelValues(resourceType, appName, namespace).Inc() +} + +// RegisterReconcileDeleteFailed increments reconcileDeleteFailedTotal +func (am *ReconcileCountMetrics) RegisterReconcileDeleteFailed(resourceType, appName, namespace string) { + am.reconcileDeleteFailedTotal.WithLabelValues(resourceType, appName, namespace).Inc() +} + +// GetReconcileAttemptCounterValue return reconcile count +func (am *ReconcileCountMetrics) GetReconcileAttemptCounterValue(resourceType, appName, namespace string) int64 { + var m = &dto.Metric{} + if err := am.reconcileAttemptTotal.WithLabelValues(resourceType, appName, namespace).Write(m); err != nil { + return 0 + } + return int64(m.Counter.GetValue()) +} diff --git a/pkg/metrics/reconcile_time_metrics.go b/pkg/metrics/reconcile_time_metrics.go new file mode 100644 index 000000000..06ac30352 --- /dev/null +++ b/pkg/metrics/reconcile_time_metrics.go @@ -0,0 +1,103 @@ +// Copyright 2021 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Package metrics to define all prometheus metric methods +package metrics + +import ( + "strconv" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +// ReconcileTimeMetrics holds reconcile time metrics +type ReconcileTimeMetrics struct { + reconcileTimeSeconds *prometheus.GaugeVec + reconcileDeployTimeSeconds *prometheus.GaugeVec + reconcileFetchTimeSeconds *prometheus.GaugeVec + reconcileTemplateTimeSeconds *prometheus.GaugeVec +} + +var ( + timeMetricsOnce sync.Once +) + +// NewReconcileTimeMetrics creates ReconcileTimeMetrics object +func NewReconcileTimeMetrics() *ReconcileTimeMetrics { + const ( + metricNamespace = "kappctrl" + resourceTypeLabel = "controller" + resourceNameLabel = "name" + firstReconcileLabel = "firstReconcile" + namespaceLabel = "namespace" + ) + return &ReconcileTimeMetrics{ + reconcileTimeSeconds: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricNamespace, + Name: "reconcile_time_seconds", + Help: "Overall time taken to reconcile a CR", + }, + []string{resourceTypeLabel, resourceNameLabel, namespaceLabel, firstReconcileLabel}, + ), + reconcileFetchTimeSeconds: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricNamespace, + Name: "reconcile_fetch_time_seconds", + Help: "Time taken to perform a fetch for a CR", + }, + []string{resourceTypeLabel, resourceNameLabel, namespaceLabel, firstReconcileLabel}, + ), + reconcileTemplateTimeSeconds: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricNamespace, + Name: "reconcile_template_time_seconds", + Help: "Time taken to perform a templating for a CR", + }, + []string{resourceTypeLabel, resourceNameLabel, namespaceLabel, firstReconcileLabel}, + ), + reconcileDeployTimeSeconds: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricNamespace, + Name: "reconcile_deploy_time_seconds", + Help: "Time taken to perform a deploy for a CR", + }, + []string{resourceTypeLabel, resourceNameLabel, namespaceLabel, firstReconcileLabel}, + ), + } +} + +// RegisterAllMetrics registers reconcile time prometheus metrics. +func (tm *ReconcileTimeMetrics) RegisterAllMetrics() { + timeMetricsOnce.Do(func() { + metrics.Registry.MustRegister( + tm.reconcileTimeSeconds, + tm.reconcileFetchTimeSeconds, + tm.reconcileTemplateTimeSeconds, + tm.reconcileDeployTimeSeconds, + ) + }) +} + +// RegisterOverallTime sets overall time +func (tm *ReconcileTimeMetrics) RegisterOverallTime(resourceType, name, namespace string, firstReconcile bool, time time.Duration) { + tm.reconcileTimeSeconds.WithLabelValues(resourceType, name, namespace, strconv.FormatBool(firstReconcile)).Set(time.Seconds()) +} + +// RegisterFetchTime sets fetch time +func (tm *ReconcileTimeMetrics) RegisterFetchTime(resourceType, name, namespace string, firstReconcile bool, time time.Duration) { + tm.reconcileFetchTimeSeconds.WithLabelValues(resourceType, name, namespace, strconv.FormatBool(firstReconcile)).Set(time.Seconds()) +} + +// RegisterTemplateTime sets template time +func (tm *ReconcileTimeMetrics) RegisterTemplateTime(resourceType, name, namespace string, firstReconcile bool, time time.Duration) { + tm.reconcileTemplateTimeSeconds.WithLabelValues(resourceType, name, namespace, strconv.FormatBool(firstReconcile)).Set(time.Seconds()) +} + +// RegisterDeployTime sets deploy time +func (tm *ReconcileTimeMetrics) RegisterDeployTime(resourceType, name, namespace string, firstReconcile bool, time time.Duration) { + tm.reconcileDeployTimeSeconds.WithLabelValues(resourceType, name, namespace, strconv.FormatBool(firstReconcile)).Set(time.Seconds()) +} diff --git a/pkg/packageinstall/packageinstall.go b/pkg/packageinstall/packageinstall.go index 9745f8498..b5aba623d 100644 --- a/pkg/packageinstall/packageinstall.go +++ b/pkg/packageinstall/packageinstall.go @@ -17,6 +17,7 @@ import ( pkgclient "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apiserver/client/clientset/versioned" kcclient "github.com/vmware-tanzu/carvel-kapp-controller/pkg/client/clientset/versioned" "github.com/vmware-tanzu/carvel-kapp-controller/pkg/client/clientset/versioned/scheme" + "github.com/vmware-tanzu/carvel-kapp-controller/pkg/metrics" "github.com/vmware-tanzu/carvel-kapp-controller/pkg/reconciler" "github.com/vmware-tanzu/carvel-vendir/pkg/vendir/versions" verv1alpha1 "github.com/vmware-tanzu/carvel-vendir/pkg/vendir/versions/v1alpha1" @@ -54,6 +55,13 @@ type PackageInstallCR struct { coreClient kubernetes.Interface compInfo ComponentInfo opts Opts + + pkgMetrics *metrics.Metrics +} + +// Kind return kind of pkg install +func (pi *PackageInstallCR) Kind() string { + return "PackageInstall" } // nolint: revive @@ -62,10 +70,11 @@ type Opts struct { } func NewPackageInstallCR(model *pkgingv1alpha1.PackageInstall, log logr.Logger, - kcclient kcclient.Interface, pkgclient pkgclient.Interface, coreClient kubernetes.Interface, compInfo ComponentInfo, opts Opts) *PackageInstallCR { + kcclient kcclient.Interface, pkgclient pkgclient.Interface, coreClient kubernetes.Interface, + compInfo ComponentInfo, opts Opts, pkgMetrics *metrics.Metrics) *PackageInstallCR { return &PackageInstallCR{model: model, unmodifiedModel: model.DeepCopy(), log: log, - kcclient: kcclient, pkgclient: pkgclient, coreClient: coreClient, compInfo: compInfo, opts: opts} + kcclient: kcclient, pkgclient: pkgclient, coreClient: coreClient, compInfo: compInfo, opts: opts, pkgMetrics: pkgMetrics} } func (pi *PackageInstallCR) Reconcile() (reconcile.Result, error) { @@ -74,6 +83,8 @@ func (pi *PackageInstallCR) Reconcile() (reconcile.Result, error) { func(st kcv1alpha1.GenericStatus) { pi.model.Status.GenericStatus = st }, } + pi.pkgMetrics.ReconcileCountMetrics.InitMetrics(pi.Kind(), pi.model.Name, pi.model.Namespace) + var result reconcile.Result var err error @@ -100,6 +111,14 @@ func (pi *PackageInstallCR) Reconcile() (reconcile.Result, error) { func (pi *PackageInstallCR) reconcile(modelStatus *reconciler.Status) (reconcile.Result, error) { pi.log.Info("Reconciling") + pi.pkgMetrics.ReconcileCountMetrics.RegisterReconcileAttempt(pi.Kind(), pi.model.Name, pi.model.Namespace) + + reconcileStartTime := time.Now() + pi.pkgMetrics.IsFirstReconcile = pi.pkgMetrics.ReconcileCountMetrics.GetReconcileAttemptCounterValue(pi.Kind(), pi.model.Name, pi.model.Namespace) == 1 + defer func() { + pi.pkgMetrics.ReconcileTimeMetrics.RegisterOverallTime(pi.Kind(), pi.model.Name, pi.model.Namespace, + pi.pkgMetrics.IsFirstReconcile, time.Since(reconcileStartTime)) + }() err := pi.blockDeletion() if err != nil { diff --git a/pkg/packageinstall/packageinstall_deletion_test.go b/pkg/packageinstall/packageinstall_deletion_test.go index 0ef39fc80..ba8ab4d7d 100644 --- a/pkg/packageinstall/packageinstall_deletion_test.go +++ b/pkg/packageinstall/packageinstall_deletion_test.go @@ -12,6 +12,7 @@ import ( pkgingv1alpha1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/packaging/v1alpha1" fakeapiserver "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apiserver/client/clientset/versioned/fake" fakekappctrl "github.com/vmware-tanzu/carvel-kapp-controller/pkg/client/clientset/versioned/fake" + "github.com/vmware-tanzu/carvel-kapp-controller/pkg/metrics" versions "github.com/vmware-tanzu/carvel-vendir/pkg/vendir/versions/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" @@ -65,7 +66,9 @@ func Test_PackageInstallDeletion(t *testing.T) { appClient := fakekappctrl.NewSimpleClientset(pkgInstall, existingApp) coreClient := fake.NewSimpleClientset() - ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, coreClient, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}) + ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, coreClient, + FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}, + metrics.NewMetrics()) _, err := ip.Reconcile() assert.Nil(t, err) diff --git a/pkg/packageinstall/packageinstall_downgrade_test.go b/pkg/packageinstall/packageinstall_downgrade_test.go index 7810d6f32..be0265a43 100644 --- a/pkg/packageinstall/packageinstall_downgrade_test.go +++ b/pkg/packageinstall/packageinstall_downgrade_test.go @@ -14,6 +14,7 @@ import ( datapkgingv1alpha1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apiserver/apis/datapackaging/v1alpha1" fakeapiserver "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apiserver/client/clientset/versioned/fake" fakekappctrl "github.com/vmware-tanzu/carvel-kapp-controller/pkg/client/clientset/versioned/fake" + "github.com/vmware-tanzu/carvel-kapp-controller/pkg/metrics" versions "github.com/vmware-tanzu/carvel-vendir/pkg/vendir/versions/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" @@ -98,7 +99,9 @@ func Test_PackageInstallVersionDowngrades(t *testing.T) { GitVersion: "v0.20.0", } - ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}) + ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, fakek8s, + FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}, + metrics.NewMetrics()) _, err := ip.Reconcile() assert.Nil(t, err) @@ -147,7 +150,9 @@ func Test_PackageInstallVersionDowngrades(t *testing.T) { GitVersion: "v0.20.0", } - ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}) + ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, fakek8s, + FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}, + metrics.NewMetrics()) _, err := ip.Reconcile() assert.Nil(t, err) @@ -196,7 +201,9 @@ func Test_PackageInstallVersionDowngrades(t *testing.T) { GitVersion: "v0.20.0", } - ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}) + ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, fakek8s, + FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}, + metrics.NewMetrics()) _, err := ip.Reconcile() assert.Nil(t, err) @@ -251,7 +258,9 @@ func Test_PackageInstallVersionDowngrades(t *testing.T) { GitVersion: "v0.20.0", } - ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}) + ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, fakek8s, + FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}, + metrics.NewMetrics()) _, err := ip.Reconcile() assert.Nil(t, err) diff --git a/pkg/packageinstall/packageinstall_test.go b/pkg/packageinstall/packageinstall_test.go index 174eb302b..7a5c576e9 100644 --- a/pkg/packageinstall/packageinstall_test.go +++ b/pkg/packageinstall/packageinstall_test.go @@ -16,6 +16,7 @@ import ( datapkgingv1alpha1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apiserver/apis/datapackaging/v1alpha1" fakeapiserver "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apiserver/client/clientset/versioned/fake" fakekappctrl "github.com/vmware-tanzu/carvel-kapp-controller/pkg/client/clientset/versioned/fake" + "github.com/vmware-tanzu/carvel-kapp-controller/pkg/metrics" versions "github.com/vmware-tanzu/carvel-vendir/pkg/vendir/versions/v1alpha1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -402,7 +403,9 @@ func Test_PlaceHolderSecretCreated_WhenPackageHasNoSecretRef(t *testing.T) { GitVersion: "v0.20.0", } - ip := NewPackageInstallCR(model, log, fakekctrl, fakePkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}) + ip := NewPackageInstallCR(model, log, fakekctrl, fakePkgClient, fakek8s, + FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}, + metrics.NewMetrics()) _, err := ip.Reconcile() assert.Nil(t, err) @@ -481,7 +484,9 @@ func Test_PlaceHolderSecretsCreated_WhenPackageHasMultipleFetchStages(t *testing GitVersion: "v0.20.0", } - ip := NewPackageInstallCR(model, log, fakekctrl, fakePkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}) + ip := NewPackageInstallCR(model, log, fakekctrl, fakePkgClient, fakek8s, + FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}, + metrics.NewMetrics()) _, err := ip.Reconcile() assert.Nil(t, err) @@ -571,7 +576,9 @@ func Test_PlaceHolderSecretsNotCreated_WhenFetchStagesHaveSecrets(t *testing.T) GitVersion: "v0.20.0", } - ip := NewPackageInstallCR(model, log, fakekctrl, fakePkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}) + ip := NewPackageInstallCR(model, log, fakekctrl, fakePkgClient, fakek8s, + FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}, + metrics.NewMetrics()) _, err := ip.Reconcile() assert.Nil(t, err) @@ -648,7 +655,9 @@ func Test_PlaceHolderSecretCreated_WhenPackageInstallUpdated(t *testing.T) { fakekctrl := fakekappctrl.NewSimpleClientset(model, existingApp) fakek8s := fake.NewSimpleClientset() - ip := NewPackageInstallCR(model, log, fakekctrl, fakePkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}) + ip := NewPackageInstallCR(model, log, fakekctrl, fakePkgClient, fakek8s, + FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}, + metrics.NewMetrics()) // mock the kubernetes server version fakeDiscovery, _ := fakek8s.Discovery().(*fakediscovery.FakeDiscovery) diff --git a/pkg/packageinstall/reconciler.go b/pkg/packageinstall/reconciler.go index 451106abd..aacf59d7d 100644 --- a/pkg/packageinstall/reconciler.go +++ b/pkg/packageinstall/reconciler.go @@ -14,6 +14,7 @@ import ( pkgclient "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apiserver/client/clientset/versioned" kcclient "github.com/vmware-tanzu/carvel-kapp-controller/pkg/client/clientset/versioned" kcconfig "github.com/vmware-tanzu/carvel-kapp-controller/pkg/config" + "github.com/vmware-tanzu/carvel-kapp-controller/pkg/metrics" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -33,12 +34,13 @@ type Reconciler struct { compInfo ComponentInfo log logr.Logger kcConfig *kcconfig.Config + pkgMetrics *metrics.Metrics } // NewReconciler is the constructor for the Reconciler struct func NewReconciler(kcClient kcclient.Interface, pkgClient pkgclient.Interface, coreClient kubernetes.Interface, pkgToPkgInstallHandler *PackageInstallVersionHandler, - log logr.Logger, compInfo ComponentInfo, kcConfig *kcconfig.Config) *Reconciler { + log logr.Logger, compInfo ComponentInfo, kcConfig *kcconfig.Config, pkgMetrics *metrics.Metrics) *Reconciler { return &Reconciler{kcClient: kcClient, pkgClient: pkgClient, @@ -47,6 +49,7 @@ func NewReconciler(kcClient kcclient.Interface, pkgClient pkgclient.Interface, compInfo: compInfo, log: log, kcConfig: kcConfig, + pkgMetrics: pkgMetrics, } } @@ -88,5 +91,6 @@ func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) ( return reconcile.Result{}, err } - return NewPackageInstallCR(existingPkgInstall, log, r.kcClient, r.pkgClient, r.coreClient, r.compInfo, Opts{DefaultSyncPeriod: r.kcConfig.PackageInstallDefaultSyncPeriod()}).Reconcile() + return NewPackageInstallCR(existingPkgInstall, log, r.kcClient, r.pkgClient, r.coreClient, r.compInfo, + Opts{DefaultSyncPeriod: r.kcConfig.PackageInstallDefaultSyncPeriod()}, r.pkgMetrics).Reconcile() } diff --git a/pkg/pkgrepository/app.go b/pkg/pkgrepository/app.go index a6951fade..fd94fd3c3 100644 --- a/pkg/pkgrepository/app.go +++ b/pkg/pkgrepository/app.go @@ -8,6 +8,7 @@ import ( "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/kappctrl/v1alpha1" "github.com/vmware-tanzu/carvel-kapp-controller/pkg/deploy" "github.com/vmware-tanzu/carvel-kapp-controller/pkg/fetch" + "github.com/vmware-tanzu/carvel-kapp-controller/pkg/metrics" "github.com/vmware-tanzu/carvel-kapp-controller/pkg/reftracker" "github.com/vmware-tanzu/carvel-kapp-controller/pkg/template" types "k8s.io/apimachinery/pkg/types" @@ -29,6 +30,8 @@ type App struct { templateFactory template.Factory deployFactory deploy.Factory + appMetrics *metrics.Metrics + log logr.Logger pendingStatusUpdate bool @@ -36,15 +39,19 @@ type App struct { } // NewApp creates a new instance of an App based on v1alpha1.App -func NewApp(app v1alpha1.App, hooks Hooks, fetchFactory fetch.Factory, templateFactory template.Factory, deployFactory deploy.Factory, log logr.Logger, pkgRepoUID types.UID) *App { +func NewApp(app v1alpha1.App, hooks Hooks, fetchFactory fetch.Factory, templateFactory template.Factory, deployFactory deploy.Factory, + log logr.Logger, appMetrics *metrics.Metrics, pkgRepoUID types.UID) *App { return &App{app: app, appPrev: *(app.DeepCopy()), hooks: hooks, fetchFactory: fetchFactory, templateFactory: templateFactory, - deployFactory: deployFactory, log: log, pkgRepoUID: pkgRepoUID} + deployFactory: deployFactory, appMetrics: appMetrics, log: log, pkgRepoUID: pkgRepoUID} } func (a *App) Name() string { return a.app.Name } func (a *App) Namespace() string { return a.app.Namespace } +// Kind return kind of pkg repo +func (a *App) Kind() string { return "PackageRepository" } + func (a *App) Status() v1alpha1.AppStatus { return a.app.Status } func (a *App) blockDeletion() error { return a.hooks.BlockDeletion() } diff --git a/pkg/pkgrepository/app_factory.go b/pkg/pkgrepository/app_factory.go index f3695baf9..930d29cf3 100644 --- a/pkg/pkgrepository/app_factory.go +++ b/pkg/pkgrepository/app_factory.go @@ -16,6 +16,7 @@ import ( "github.com/vmware-tanzu/carvel-kapp-controller/pkg/fetch" "github.com/vmware-tanzu/carvel-kapp-controller/pkg/kubeconfig" "github.com/vmware-tanzu/carvel-kapp-controller/pkg/memdir" + "github.com/vmware-tanzu/carvel-kapp-controller/pkg/metrics" "github.com/vmware-tanzu/carvel-kapp-controller/pkg/template" "k8s.io/client-go/kubernetes" ) @@ -24,6 +25,7 @@ import ( type AppFactory struct { CoreClient kubernetes.Interface AppClient kcclient.Interface + AppMetrics *metrics.Metrics KcConfig *config.Config CmdRunner exec.CmdRunner Kubeconf *kubeconfig.Kubeconfig @@ -39,5 +41,5 @@ func (f *AppFactory) NewCRDPackageRepo(app *kcv1alpha1.App, pkgr *pkgv1alpha1.Pa fetchFactory := fetch.NewFactory(f.CoreClient, vendirOpts, f.CmdRunner) templateFactory := template.NewFactory(f.CoreClient, fetchFactory, false, f.CmdRunner) deployFactory := deploy.NewFactory(f.CoreClient, f.Kubeconf, nil, f.CmdRunner, log) - return NewCRDApp(app, pkgr, log, f.AppClient, fetchFactory, templateFactory, deployFactory) + return NewCRDApp(app, pkgr, log, f.AppClient, f.AppMetrics, fetchFactory, templateFactory, deployFactory) } diff --git a/pkg/pkgrepository/app_reconcile.go b/pkg/pkgrepository/app_reconcile.go index 288f45d05..f5ce30e2e 100644 --- a/pkg/pkgrepository/app_reconcile.go +++ b/pkg/pkgrepository/app_reconcile.go @@ -21,6 +21,8 @@ func (a *App) Reconcile(force bool) (reconcile.Result, error) { var err error + a.appMetrics.ReconcileCountMetrics.InitMetrics(a.Kind(), a.Name(), a.Namespace()) + switch { case a.app.Spec.Paused: a.log.Info("PackageRepository is paused, not reconciling") @@ -90,6 +92,13 @@ func (a *App) reconcileDeploy() error { } func (a *App) reconcileFetchTemplateDeploy() exec.CmdRunResult { + reconcileStartTime := time.Now() + a.appMetrics.IsFirstReconcile = a.appMetrics.ReconcileCountMetrics.GetReconcileAttemptCounterValue(a.Kind(), a.Name(), a.Namespace()) == 1 + defer func() { + a.appMetrics.ReconcileTimeMetrics.RegisterOverallTime(a.Kind(), a.Name(), a.Namespace(), a.appMetrics.IsFirstReconcile, + time.Since(reconcileStartTime)) + }() + tmpDir := memdir.NewTmpDir("fetch-template-deploy") err := tmpDir.Create() @@ -116,6 +125,9 @@ func (a *App) reconcileFetchTemplateDeploy() exec.CmdRunResult { UpdatedAt: metav1.NewTime(time.Now().UTC()), } + a.appMetrics.ReconcileTimeMetrics.RegisterFetchTime(a.Kind(), a.Name(), a.Namespace(), a.appMetrics.IsFirstReconcile, + a.app.Status.Fetch.UpdatedAt.Sub(a.app.Status.Fetch.StartedAt.Time)) + err := a.updateStatus("marking fetch completed") if err != nil { return exec.NewCmdRunResultWithErr(err) @@ -126,6 +138,8 @@ func (a *App) reconcileFetchTemplateDeploy() exec.CmdRunResult { } } + templateStartTime := time.Now() + tplResult := a.template(assetsPath) a.app.Status.Template = &v1alpha1.AppStatusTemplate{ @@ -135,6 +149,9 @@ func (a *App) reconcileFetchTemplateDeploy() exec.CmdRunResult { UpdatedAt: metav1.NewTime(time.Now().UTC()), } + a.appMetrics.ReconcileTimeMetrics.RegisterTemplateTime(a.Kind(), a.Name(), a.Namespace(), a.appMetrics.IsFirstReconcile, + a.app.Status.Template.UpdatedAt.Sub(templateStartTime)) + err = a.updateStatus("marking template completed") if err != nil { return exec.NewCmdRunResultWithErr(err) @@ -162,6 +179,9 @@ func (a *App) updateLastDeploy(result exec.CmdRunResult) exec.CmdRunResult { UpdatedAt: metav1.NewTime(time.Now().UTC()), } + a.appMetrics.ReconcileTimeMetrics.RegisterDeployTime(a.Kind(), a.Name(), a.Namespace(), a.appMetrics.IsFirstReconcile, + a.Status().Deploy.UpdatedAt.Sub(a.Status().Deploy.StartedAt.Time)) + a.updateStatus("marking last deploy") return result @@ -193,6 +213,8 @@ func (a *App) setReconciling() { Status: corev1.ConditionTrue, }) + a.appMetrics.ReconcileCountMetrics.RegisterReconcileAttempt(a.Kind(), a.Name(), a.Namespace()) + a.app.Status.FriendlyDescription = "Reconciling" } diff --git a/pkg/pkgrepository/app_reconcile_test.go b/pkg/pkgrepository/app_reconcile_test.go index 9962d1820..6306b437c 100644 --- a/pkg/pkgrepository/app_reconcile_test.go +++ b/pkg/pkgrepository/app_reconcile_test.go @@ -14,6 +14,7 @@ import ( "github.com/vmware-tanzu/carvel-kapp-controller/pkg/exec" "github.com/vmware-tanzu/carvel-kapp-controller/pkg/fetch" "github.com/vmware-tanzu/carvel-kapp-controller/pkg/kubeconfig" + "github.com/vmware-tanzu/carvel-kapp-controller/pkg/metrics" "github.com/vmware-tanzu/carvel-kapp-controller/pkg/template" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -46,7 +47,8 @@ func Test_NoInspectReconcile_IfNoDeployAttempted(t *testing.T) { deployFac := deploy.NewFactory(k8scs, kubeconfig.NewKubeconfig(k8scs, log), nil, exec.NewPlainCmdRunner(), log) pkgr := v1alpha12.PackageRepository{} - crdApp := NewCRDApp(&app, &pkgr, log, kappcs, fetchFac, tmpFac, deployFac) + crdApp := NewCRDApp(&app, &pkgr, log, kappcs, metrics.NewMetrics(), + fetchFac, tmpFac, deployFac) _, err := crdApp.Reconcile(false) if err != nil { t.Fatalf("Unexpected error with reconciling: %v", err) @@ -102,7 +104,8 @@ func Test_TemplateError_DisplayedInStatus_UsefulErrorMessageProperty(t *testing. deployFac := deploy.NewFactory(k8scs, kubeconfig.NewKubeconfig(k8scs, log), nil, exec.NewPlainCmdRunner(), log) pkgr := v1alpha12.PackageRepository{} - crdApp := NewCRDApp(&app, &pkgr, log, kappcs, fetchFac, tmpFac, deployFac) + crdApp := NewCRDApp(&app, &pkgr, log, kappcs, metrics.NewMetrics(), + fetchFac, tmpFac, deployFac) _, err := crdApp.Reconcile(false) if err != nil { t.Fatalf("Unexpected error with reconciling: %v", err) @@ -158,7 +161,8 @@ func TestInvalidPackageRepositoryFormat(t *testing.T) { deployFac := deploy.NewFactory(k8scs, kubeconfig.NewKubeconfig(k8scs, log), nil, exec.NewPlainCmdRunner(), log) pkgr := v1alpha12.PackageRepository{} - crdApp := NewCRDApp(&app, &pkgr, log, kappcs, fetchFac, tmpFac, deployFac) + crdApp := NewCRDApp(&app, &pkgr, log, kappcs, metrics.NewMetrics(), + fetchFac, tmpFac, deployFac) _, err := crdApp.Reconcile(false) if err != nil { t.Fatalf("Unexpected error with reconciling: %v", err) diff --git a/pkg/pkgrepository/crd_app.go b/pkg/pkgrepository/crd_app.go index c6bdbabc8..a85c56fd3 100644 --- a/pkg/pkgrepository/crd_app.go +++ b/pkg/pkgrepository/crd_app.go @@ -13,6 +13,7 @@ import ( kcclient "github.com/vmware-tanzu/carvel-kapp-controller/pkg/client/clientset/versioned" "github.com/vmware-tanzu/carvel-kapp-controller/pkg/deploy" "github.com/vmware-tanzu/carvel-kapp-controller/pkg/fetch" + "github.com/vmware-tanzu/carvel-kapp-controller/pkg/metrics" "github.com/vmware-tanzu/carvel-kapp-controller/pkg/reftracker" "github.com/vmware-tanzu/carvel-kapp-controller/pkg/template" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -28,7 +29,7 @@ type CRDApp struct { } func NewCRDApp(appModel *kcv1alpha1.App, packageRepo *pkgingv1alpha1.PackageRepository, log logr.Logger, - appClient kcclient.Interface, fetchFactory fetch.Factory, + appClient kcclient.Interface, appMetrics *metrics.Metrics, fetchFactory fetch.Factory, templateFactory template.Factory, deployFactory deploy.Factory) *CRDApp { crdApp := &CRDApp{appModel: appModel, pkgrModel: packageRepo, log: log, appClient: appClient} @@ -37,7 +38,7 @@ func NewCRDApp(appModel *kcv1alpha1.App, packageRepo *pkgingv1alpha1.PackageRepo BlockDeletion: crdApp.blockDeletion, UnblockDeletion: crdApp.unblockDeletion, UpdateStatus: crdApp.updateStatus, - }, fetchFactory, templateFactory, deployFactory, log, packageRepo.UID) + }, fetchFactory, templateFactory, deployFactory, log, appMetrics, packageRepo.UID) return crdApp } diff --git a/test/e2e/kappcontroller/metrics_test.go b/test/e2e/kappcontroller/metrics_test.go new file mode 100644 index 000000000..ab6526a38 --- /dev/null +++ b/test/e2e/kappcontroller/metrics_test.go @@ -0,0 +1,203 @@ +// Copyright 2021 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package kappcontroller + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/vmware-tanzu/carvel-kapp-controller/test/e2e" +) + + +func TestPrometheusMetrics(t *testing.T) { + env := e2e.BuildEnv(t) + logger := e2e.Logger{} + kapp := e2e.Kapp{t, env.Namespace, logger} + sas := e2e.ServiceAccounts{env.Namespace} + + pkgRepoYAML := fmt.Sprintf(` +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageRepository +metadata: + name: minimal-repo.tanzu.carvel.dev + namespace: %s + annotations: + kapp.k14s.io/disable-original: "" +spec: + fetch: + inline: + paths: + + packages/pkg.test.carvel.dev/pkg.test.carvel.dev.0.0.0.yml: | + --- + apiVersion: data.packaging.carvel.dev/v1alpha1 + kind: Package + metadata: + name: pkg.test.carvel.dev.0.0.0 + spec: + refName: pkg.test.carvel.dev + version: 0.0.0 + template: + spec: {} +`, env.Namespace) + + installPkgYAML := fmt.Sprintf(`--- +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: PackageMetadata +metadata: + name: pkg.test.carvel.dev + namespace: %[1]s +spec: + # This is the name we want to reference in resources such as PackageInstall. + displayName: "Test PackageMetadata in repo" + shortDescription: "PackageMetadata used for testing" + longDescription: "A longer, more detailed description of what the package contains and what it is for" + providerName: Carvel + maintainers: + - name: carvel + categories: + - testing + supportDescription: "Description of support provided for the package" +--- +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: Package +metadata: + name: pkg.test.carvel.dev.1.0.0 + namespace: %[1]s +spec: + refName: pkg.test.carvel.dev + version: 1.0.0 + licenses: + - Apache 2.0 + capactiyRequirementsDescription: "cpu: 1,RAM: 2, Disk: 3" + releaseNotes: | + - Introduce simple-app package + releasedAt: 2021-05-05T18:57:06Z + template: + spec: + fetch: + - imgpkgBundle: + image: k8slt/kctrl-example-pkg:v1.0.0 + template: + - ytt: {} + - kbld: + paths: + - "-" + - ".imgpkg/images.yml" + deploy: + - kapp: + inspect: {} +--- +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageInstall +metadata: + name: instl-pkg-test + namespace: %[1]s + annotations: + kapp.k14s.io/change-group: kappctrl-e2e.k14s.io/packageinstalls +spec: + serviceAccountName: kappctrl-e2e-ns-sa + packageRef: + refName: pkg.test.carvel.dev + versionSelection: + constraints: 1.0.0 + values: + - secretRef: + name: pkg-demo-values +--- +apiVersion: v1 +kind: Secret +metadata: + name: pkg-demo-values +stringData: + values.yml: | + hello_msg: "hi" +`, env.Namespace) + sas.ForNamespaceYAML() + + cleanUp := func() { + kapp.Run([]string{"delete", "-a", "simple-app.app"}) + kapp.Run([]string{"delete", "-a", "simple-app"}) + kapp.Run([]string{"delete", "-a", "default-ns-rbac"}) + kapp.Run([]string{"delete", "-a", "instl-pkg-test"}) + kapp.Run([]string{"delete", "-a", "minimal-repo.tanzu.carvel.dev"}) + } + cleanUp() + defer cleanUp() + + ctx, cancel := context.WithCancel(context.Background()) + + var wg sync.WaitGroup + wg.Add(1) + + // port-forwarding goroutine + go func() { + defer wg.Done() + portForward(ctx) + }() + + // Allow some time for port-forwarding + time.Sleep(2 * time.Second) + + kapp.Run([]string{"deploy", "-a", "default-ns-rbac", "-f", + "https://raw.githubusercontent.com/carvel-dev/kapp-controller/develop/examples/rbac/default-ns.yml"}) + + kapp.Run([]string{"deploy", "-a", "simple-app", "-f", + "https://raw.githubusercontent.com/k14s/kapp-controller/develop/examples/simple-app-git/1.yml"}) + + kapp.RunWithOpts([]string{"deploy", "-a", "minimal-repo.tanzu.carvel.dev", "-f", "-"}, e2e.RunOpts{StdinReader: strings.NewReader(pkgRepoYAML)}) + + kapp.RunWithOpts([]string{"deploy", "-a", "instl-pkg-test", "-f", "-"}, e2e.RunOpts{StdinReader: strings.NewReader(installPkgYAML)}) + + t.Logf("Hitting URL") + + resp, err := http.Get("http://localhost:8080/metrics") + assert.NoError(t, err) + defer resp.Body.Close() + + // Check if the response is successful + assert.Equal(t, http.StatusOK, resp.StatusCode) + + bodyBytes, err := io.ReadAll(resp.Body) + + response := string(bodyBytes) + + bodyContains := assert.Contains(t, response, "kappctrl_reconcile_deploy_time_seconds") && + assert.Contains(t, response, "kappctrl_reconcile_fetch_time_seconds") && + assert.Contains(t, response, "kappctrl_reconcile_template_time_seconds") && + assert.Contains(t, response, "kappctrl_reconcile_time_seconds") + + assert.True(t, bodyContains) + + // Stop port-forwarding by canceling the context + cancel() + + // Wait for the port-forwarding goroutine to complete + wg.Wait() +} + +func portForward(ctx context.Context) { + cmd := exec.CommandContext(ctx, "kubectl", "port-forward", "svc/packaging-api", "8080", "-n", "kapp-controller") + + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + err := cmd.Run() + if err != nil { + if ctx.Err() == context.Canceled { + fmt.Println("Port-forwarding stopped by context cancellation.") + } else { + fmt.Printf("Error running kubectl port-forward: %v\n", err) + } + } +} \ No newline at end of file