Enable serving supervisor metrics

.golangci.json

@@ -10,20 +10,20 @@
 		]
 	},
 	"run": {
-		"skip-dirs": [
+		"deadline": "5m"
+	},
+	"issues": {
+		"exclude-dirs": [
 			"build",
 			"contrib",
 			"manifests",
 			"package",
 			"scripts",
 			"vendor"
 		],
-		"skip-files": [
+		"exclude-files": [
 			"/zz_generated_"
 		],
-		"deadline": "5m"
-	},
-	"issues": {
 		"exclude-rules": [
 			{
 				"linters": "typecheck",
@@ -43,4 +43,4 @@
 			}
 		]
 	}
-}
+}

pkg/agent/config/config.go

@@ -512,12 +512,14 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 		SELinux:                  envInfo.EnableSELinux,
 		ContainerRuntimeEndpoint: envInfo.ContainerRuntimeEndpoint,
 		ImageServiceEndpoint:     envInfo.ImageServiceEndpoint,
+		EnablePProf:              envInfo.EnablePProf,
 		EmbeddedRegistry:         controlConfig.EmbeddedRegistry,
 		FlannelBackend:           controlConfig.FlannelBackend,
 		FlannelIPv6Masq:          controlConfig.FlannelIPv6Masq,
 		FlannelExternalIP:        controlConfig.FlannelExternalIP,
 		EgressSelectorMode:       controlConfig.EgressSelectorMode,
 		ServerHTTPSPort:          controlConfig.HTTPSPort,
+		SupervisorMetrics:        controlConfig.SupervisorMetrics,
 		Token:                    info.String(),
 	}
 	nodeConfig.FlannelIface = flannelIface
@@ -580,13 +582,18 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	nodeConfig.Containerd.Template = filepath.Join(envInfo.DataDir, "agent", "etc", "containerd", "config.toml.tmpl")
 	nodeConfig.Certificate = servingCert
 
-	nodeConfig.AgentConfig.NodeIPs = nodeIPs
-	listenAddress, _, _, err := util.GetDefaultAddresses(nodeIPs[0])
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot configure IPv4/IPv6 node-ip")
+	if envInfo.BindAddress != "" {
+		nodeConfig.AgentConfig.ListenAddress = envInfo.BindAddress
+	} else {
+		listenAddress, _, _, err := util.GetDefaultAddresses(nodeIPs[0])
+		if err != nil {
+			return nil, errors.Wrap(err, "cannot configure IPv4/IPv6 node-ip")
+		}
+		nodeConfig.AgentConfig.ListenAddress = listenAddress
 	}
+
 	nodeConfig.AgentConfig.NodeIP = nodeIPs[0].String()
-	nodeConfig.AgentConfig.ListenAddress = listenAddress
+	nodeConfig.AgentConfig.NodeIPs = nodeIPs
 	nodeConfig.AgentConfig.NodeExternalIPs = nodeExternalIPs
 
 	// if configured, set NodeExternalIP to the first IPv4 address, for legacy clients
@@ -677,6 +684,8 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	nodeConfig.AgentConfig.ImageCredProvConfig = envInfo.ImageCredProvConfig
 	nodeConfig.AgentConfig.DisableCCM = controlConfig.DisableCCM
 	nodeConfig.AgentConfig.DisableNPC = controlConfig.DisableNPC
+	nodeConfig.AgentConfig.MinTLSVersion = controlConfig.MinTLSVersion
+	nodeConfig.AgentConfig.CipherSuites = controlConfig.CipherSuites
 	nodeConfig.AgentConfig.Rootless = envInfo.Rootless
 	nodeConfig.AgentConfig.PodManifests = filepath.Join(envInfo.DataDir, "agent", DefaultPodManifestPath)
 	nodeConfig.AgentConfig.ProtectKernelDefaults = envInfo.ProtectKernelDefaults

pkg/agent/https/https.go

@@ -0,0 +1,106 @@
+package https
+
+import (
+	"context"
+	"net/http"
+	"strconv"
+	"sync"
+
+	"github.com/gorilla/mux"
+	"github.com/k3s-io/k3s/pkg/daemons/config"
+	"github.com/k3s-io/k3s/pkg/generated/clientset/versioned/scheme"
+	"github.com/k3s-io/k3s/pkg/util"
+	"github.com/k3s-io/k3s/pkg/version"
+	"k8s.io/apiserver/pkg/authentication/authenticator"
+	"k8s.io/apiserver/pkg/authorization/authorizer"
+	genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
+	apirequest "k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/options"
+)
+
+// RouterFunc provides a hook for components to register additional routes to a request router
+type RouterFunc func(ctx context.Context, nodeConfig *config.Node) (*mux.Router, error)
+
+var once sync.Once
+var router *mux.Router
+var err error
+
+// Start returns a router with authn/authz filters applied.
+// The first time it is called, the router is created and a new HTTPS listener is started if the handler is nil.
+// Subsequent calls will return the same router.
+func Start(ctx context.Context, nodeConfig *config.Node, runtime *config.ControlRuntime) (*mux.Router, error) {
+	once.Do(func() {
+		router = mux.NewRouter().SkipClean(true)
+		config := server.Config{}
+
+		if runtime == nil {
+			// If we do not have an existing handler, set up a new listener
+			tcp, lerr := util.ListenWithLoopback(ctx, nodeConfig.AgentConfig.ListenAddress, strconv.Itoa(nodeConfig.ServerHTTPSPort))
+			if lerr != nil {
+				err = lerr
+				return
+			}
+
+			serving := options.NewSecureServingOptions()
+			serving.Listener = tcp
+			serving.CipherSuites = nodeConfig.AgentConfig.CipherSuites
+			serving.MinTLSVersion = nodeConfig.AgentConfig.MinTLSVersion
+			serving.ServerCert = options.GeneratableKeyCert{
+				CertKey: options.CertKey{
+					CertFile: nodeConfig.AgentConfig.ServingKubeletCert,
+					KeyFile:  nodeConfig.AgentConfig.ServingKubeletKey,
+				},
+			}
+			if aerr := serving.ApplyTo(&config.SecureServing); aerr != nil {
+				err = aerr
+				return
+			}
+		} else {
+			// If we have an existing handler, wrap it
+			router.NotFoundHandler = runtime.Handler
+			runtime.Handler = router
+		}
+
+		authn := options.NewDelegatingAuthenticationOptions()
+		authn.DisableAnonymous = true
+		authn.SkipInClusterLookup = true
+		authn.ClientCert = options.ClientCertAuthenticationOptions{
+			ClientCA: nodeConfig.AgentConfig.ClientCA,
+		}
+		authn.RemoteKubeConfigFile = nodeConfig.AgentConfig.KubeConfigKubelet
+		if applyErr := authn.ApplyTo(&config.Authentication, config.SecureServing, nil); applyErr != nil {
+			err = applyErr
+			return
+		}
+
+		authz := options.NewDelegatingAuthorizationOptions()
+		authz.AlwaysAllowPaths = []string{"/v2", "/debug/pprof", "/v1-" + version.Program + "/p2p"}
+		authz.RemoteKubeConfigFile = nodeConfig.AgentConfig.KubeConfigKubelet
+		if applyErr := authz.ApplyTo(&config.Authorization); applyErr != nil {
+			err = applyErr
+			return
+		}
+
+		router.Use(filterChain(config.Authentication.Authenticator, config.Authorization.Authorizer))
+
+		if config.SecureServing != nil {
+			_, _, err = config.SecureServing.Serve(router, 0, ctx.Done())
+		}
+	})
+
+	return router, err
+}
+
+// filterChain runs the kubernetes authn/authz filter chain using the mux middleware API
+func filterChain(authn authenticator.Request, authz authorizer.Authorizer) mux.MiddlewareFunc {
+	return func(handler http.Handler) http.Handler {
+		requestInfoResolver := &apirequest.RequestInfoFactory{}
+		failedHandler := genericapifilters.Unauthorized(scheme.Codecs)
+		handler = genericapifilters.WithAuthorization(handler, authz, scheme.Codecs)
+		handler = genericapifilters.WithAuthentication(handler, authn, failedHandler, nil, nil)
+		handler = genericapifilters.WithRequestInfo(handler, requestInfoResolver)
+		handler = genericapifilters.WithCacheControl(handler)
+		return handler
+	}
+}

pkg/agent/netpol/netpol.go

@@ -19,25 +19,25 @@ import (
 
 	"github.com/cloudnativelabs/kube-router/v2/pkg/controllers/netpol"
 	"github.com/cloudnativelabs/kube-router/v2/pkg/healthcheck"
-	"github.com/cloudnativelabs/kube-router/v2/pkg/metrics"
+	krmetrics "github.com/cloudnativelabs/kube-router/v2/pkg/metrics"
 	"github.com/cloudnativelabs/kube-router/v2/pkg/options"
 	"github.com/cloudnativelabs/kube-router/v2/pkg/utils"
 	"github.com/cloudnativelabs/kube-router/v2/pkg/version"
 	"github.com/coreos/go-iptables/iptables"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
+	"github.com/k3s-io/k3s/pkg/metrics"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	v1core "k8s.io/api/core/v1"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/component-base/metrics/legacyregistry"
 )
 
 func init() {
 	// ensure that kube-router exposes metrics through the same registry used by Kubernetes components
-	metrics.DefaultRegisterer = legacyregistry.Registerer()
-	metrics.DefaultGatherer = legacyregistry.DefaultGatherer
+	krmetrics.DefaultRegisterer = metrics.DefaultRegisterer
+	krmetrics.DefaultGatherer = metrics.DefaultGatherer
 }
 
 // Run creates and starts a new instance of the kube-router network policy controller
@@ -156,7 +156,7 @@ func Run(ctx context.Context, nodeConfig *config.Node) error {
 	}
 
 	// Start kube-router metrics controller to avoid complaints about metrics heartbeat missing
-	mc, err := metrics.NewMetricsController(krConfig)
+	mc, err := krmetrics.NewMetricsController(krConfig)
 	if err != nil {
 		return nil
 	}
@@ -188,13 +188,13 @@ func Run(ctx context.Context, nodeConfig *config.Node) error {
 }
 
 // metricsRunCheck is a stub version of mc.Run() that doesn't start up a dedicated http server.
-func metricsRunCheck(mc *metrics.Controller, healthChan chan<- *healthcheck.ControllerHeartbeat, stopCh <-chan struct{}, wg *sync.WaitGroup) {
+func metricsRunCheck(mc *krmetrics.Controller, healthChan chan<- *healthcheck.ControllerHeartbeat, stopCh <-chan struct{}, wg *sync.WaitGroup) {
 	t := time.NewTicker(3 * time.Second)
 	defer wg.Done()
 
 	// register metrics for this controller
-	metrics.BuildInfo.WithLabelValues(runtime.Version(), version.Version).Set(1)
-	metrics.DefaultRegisterer.MustRegister(metrics.BuildInfo)
+	krmetrics.BuildInfo.WithLabelValues(runtime.Version(), version.Version).Set(1)
+	krmetrics.DefaultRegisterer.MustRegister(krmetrics.BuildInfo)
 
 	for {
 		healthcheck.SendHeartBeat(healthChan, "MC")

pkg/agent/run.go

@@ -27,7 +27,9 @@ import (
 	"github.com/k3s-io/k3s/pkg/daemons/agent"
 	daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/daemons/executor"
+	"github.com/k3s-io/k3s/pkg/metrics"
 	"github.com/k3s-io/k3s/pkg/nodeconfig"
+	"github.com/k3s-io/k3s/pkg/profile"
 	"github.com/k3s-io/k3s/pkg/rootless"
 	"github.com/k3s-io/k3s/pkg/spegel"
 	"github.com/k3s-io/k3s/pkg/util"
@@ -113,6 +115,18 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error {
 		}
 	}
 
+	if nodeConfig.SupervisorMetrics {
+		if err := metrics.DefaultMetrics.Start(ctx, nodeConfig); err != nil {
+			return errors.Wrap(err, "failed to serve metrics")
+		}
+	}
+
+	if nodeConfig.EnablePProf {
+		if err := profile.DefaultProfiler.Start(ctx, nodeConfig); err != nil {
+			return errors.Wrap(err, "failed to serve pprof")
+		}
+	}
+
 	if err := setupCriCtlConfig(cfg, nodeConfig); err != nil {
 		return err
 	}

pkg/certmonitor/certmonitor.go

@@ -11,6 +11,7 @@ import (
 
 	daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/daemons/control/deps"
+	"github.com/k3s-io/k3s/pkg/metrics"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/util/services"
 	"github.com/k3s-io/k3s/pkg/version"
@@ -22,18 +23,9 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/component-base/metrics/legacyregistry"
 )
 
 var (
-	// DefaultRegisterer and DefaultGatherer are the implementations of the
-	// prometheus Registerer and Gatherer interfaces that all metrics operations
-	// will use. They are variables so that packages that embed this library can
-	// replace them at runtime, instead of having to pass around specific
-	// registries.
-	DefaultRegisterer = legacyregistry.Registerer()
-	DefaultGatherer   = legacyregistry.DefaultGatherer
-
 	// Check certificates twice an hour. Kubernetes events have a TTL of 1 hour by default,
 	// so similar events should be aggregated and refreshed by the event recorder as long
 	// as they are created within the TTL period.
@@ -50,7 +42,7 @@ var (
 // Setup starts the certificate expiration monitor
 func Setup(ctx context.Context, nodeConfig *daemonconfig.Node, dataDir string) error {
 	logrus.Debugf("Starting %s with monitoring period %s", controllerName, certCheckInterval)
-	DefaultRegisterer.MustRegister(certificateExpirationSeconds)
+	metrics.DefaultRegisterer.MustRegister(certificateExpirationSeconds)
 
 	client, err := util.GetClientSet(nodeConfig.AgentConfig.KubeConfigKubelet)
 	if err != nil {

pkg/cli/agent/agent.go

@@ -1,28 +1,29 @@
 package agent
 
 import (
+	"context"
 	"crypto/tls"
-	"errors"
 	"fmt"
-	"net/http"
 	"os"
 	"path/filepath"
 	"runtime"
 
 	"github.com/gorilla/mux"
 	"github.com/k3s-io/k3s/pkg/agent"
-	"github.com/k3s-io/k3s/pkg/authenticator"
+	"github.com/k3s-io/k3s/pkg/agent/https"
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
+	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/datadir"
+	k3smetrics "github.com/k3s-io/k3s/pkg/metrics"
 	"github.com/k3s-io/k3s/pkg/proctitle"
+	"github.com/k3s-io/k3s/pkg/profile"
 	"github.com/k3s-io/k3s/pkg/spegel"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/k3s-io/k3s/pkg/vpn"
 	"github.com/rancher/wrangler/v3/pkg/signals"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
-	apiauth "k8s.io/apiserver/pkg/authentication/authenticator"
 )
 
 func Run(ctx *cli.Context) error {
@@ -108,33 +109,22 @@ func Run(ctx *cli.Context) error {
 	// Until the agent is run and retrieves config from the server, we won't know
 	// if the embedded registry is enabled. If it is not enabled, these are not
 	// used as the registry is never started.
-	conf := spegel.DefaultRegistry
-	conf.Bootstrapper = spegel.NewAgentBootstrapper(cfg.ServerURL, cfg.Token, cfg.DataDir)
-	conf.HandlerFunc = func(conf *spegel.Config, router *mux.Router) error {
-		// Create and bind a new authenticator using the configured client CA
-		authArgs := []string{"--client-ca-file=" + conf.ClientCAFile}
-		auth, err := authenticator.FromArgs(authArgs)
-		if err != nil {
-			return err
-		}
-		conf.AuthFunc = func() apiauth.Request {
-			return auth
-		}
+	registry := spegel.DefaultRegistry
+	registry.Bootstrapper = spegel.NewAgentBootstrapper(cfg.ServerURL, cfg.Token, cfg.DataDir)
+	registry.Router = func(ctx context.Context, nodeConfig *config.Node) (*mux.Router, error) {
+		return https.Start(ctx, nodeConfig, nil)
+	}
 
-		// Create a new server and listen on the configured port
-		server := &http.Server{
-			Handler: router,
-			Addr:    ":" + conf.RegistryPort,
-			TLSConfig: &tls.Config{
-				ClientAuth: tls.RequestClientCert,
-			},
-		}
-		go func() {
-			if err := server.ListenAndServeTLS(conf.ServerCertFile, conf.ServerKeyFile); err != nil && !errors.Is(err, http.ErrServerClosed) {
-				logrus.Fatalf("registry server failed: %v", err)
-			}
-		}()
-		return nil
+	// same deal for metrics - these are not used if the extra metrics listener is not enabled.
+	metrics := k3smetrics.DefaultMetrics
+	metrics.Router = func(ctx context.Context, nodeConfig *config.Node) (*mux.Router, error) {
+		return https.Start(ctx, nodeConfig, nil)
+	}
+
+	// and for pprof as well
+	pprof := profile.DefaultProfiler
+	pprof.Router = func(ctx context.Context, nodeConfig *config.Node) (*mux.Router, error) {
+		return https.Start(ctx, nodeConfig, nil)
 	}
 
 	return agent.Run(contextCtx, cfg)