Allow halting on certificate errors

When certificate errors are encountered, the fix is usually to restart
the affected pod. To allow this to happen automatically, add a
configuration setting for the gateway agent; extend the mechanism to
the route agent.

The setting is disabled by default; it will be enabled by default by
the operator.

Signed-off-by: Stephen Kitt <skitt@redhat.com>

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/projectcalico/api v0.0.0-20230602153125-fb7148692637
 	github.com/prometheus-community/pro-bing v0.3.0
 	github.com/prometheus/client_golang v1.17.0
-	github.com/submariner-io/admiral v0.16.0-m4.0.20231010063642-6d040ab176ec
+	github.com/submariner-io/admiral v0.16.0-m4.0.20231024075740-7ca36d2067a5
 	github.com/submariner-io/shipyard v0.16.0-m4.0.20231017114407-11b7ad52c6a4
 	github.com/uw-labs/lichen v0.1.7
 	github.com/vishvananda/netlink v1.2.1-beta.2
@@ -101,7 +101,6 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/klog v1.0.0 // indirect
 	k8s.io/klog/v2 v2.100.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect

go.sum

@@ -504,8 +504,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/submariner-io/admiral v0.16.0-m4.0.20231010063642-6d040ab176ec h1:7ogpjwi+XpSPhLf8FNWsREXVZeOK1csnUSu4MLaCXG0=
-github.com/submariner-io/admiral v0.16.0-m4.0.20231010063642-6d040ab176ec/go.mod h1:Zb/vxLUvvPivyyL3wSYadlyWRGNc5hRuk5NRCGHlt2g=
+github.com/submariner-io/admiral v0.16.0-m4.0.20231024075740-7ca36d2067a5 h1:r/wA9Suzyfxpt8LMnBhZDjXLzIBpjAwcBt5FLL//cMU=
+github.com/submariner-io/admiral v0.16.0-m4.0.20231024075740-7ca36d2067a5/go.mod h1:bfpKC5z/0nOVjflOmGUkKirF3bOv5mZdRp9kOvBulAc=
 github.com/submariner-io/shipyard v0.16.0-m4.0.20231017114407-11b7ad52c6a4 h1:toajDp31eWHV2cL+oFdKVdZrrcojX7EC5HcpG5/Qjj8=
 github.com/submariner-io/shipyard v0.16.0-m4.0.20231017114407-11b7ad52c6a4/go.mod h1:1zPFbxQbgZZXvV2rukb1EliGog4+OlAEhbU5aLHwpXA=
 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
@@ -804,7 +804,6 @@ k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8
 k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
-k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
 k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=

main.go

@@ -20,21 +20,19 @@ package main
 
 import (
 	"context"
-	"crypto/x509"
 	"errors"
 	"flag"
 	"net/http"
 	"net/http/pprof"
-	"sync/atomic"
 	"time"
 
 	"github.com/kelseyhightower/envconfig"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/submariner-io/admiral/pkg/log"
 	"github.com/submariner-io/admiral/pkg/log/kzerolog"
 	"github.com/submariner-io/admiral/pkg/names"
-	"github.com/submariner-io/admiral/pkg/resource"
 	"github.com/submariner-io/admiral/pkg/syncer/broker"
+	"github.com/submariner-io/admiral/pkg/util"
 	admversion "github.com/submariner-io/admiral/pkg/version"
 	"github.com/submariner-io/admiral/pkg/watcher"
 	subv1 "github.com/submariner-io/submariner/pkg/apis/submariner.io/v1"
@@ -44,7 +42,6 @@ import (
 	"github.com/submariner-io/submariner/pkg/natdiscovery"
 	"github.com/submariner-io/submariner/pkg/types"
 	"github.com/submariner-io/submariner/pkg/versions"
-	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
@@ -57,6 +54,7 @@ var (
 	localMasterURL  string
 	localKubeconfig string
 	showVersion     = false
+	logger          = log.Logger{Logger: logf.Log.WithName("main")}
 )
 
 func init() {
@@ -74,11 +72,6 @@ type leaderConfig struct {
 
 const leadershipConfigEnvPrefix = "leadership"
 
-var (
-	logger             = log.Logger{Logger: logf.Log.WithName("main")}
-	lastBadCertificate atomic.Value
-)
-
 func main() {
 	kzerolog.AddFlags(nil)
 	flag.Parse()
@@ -105,18 +98,7 @@ func main() {
 
 	var err error
 
-	//nolint:reassign // We need to reassign ErrorHandlers to register our handler
-	utilruntime.ErrorHandlers = append(utilruntime.ErrorHandlers, func(err error) {
-		var unknownAuthorityError x509.UnknownAuthorityError
-		if errors.As(err, &unknownAuthorityError) && lastBadCertificate.Swap(unknownAuthorityError.Cert) != unknownAuthorityError.Cert {
-			logger.Errorf(err, "Certificate error: %s", resource.ToJSON(err))
-		}
-		var certificateInvalidError x509.CertificateInvalidError
-		if errors.As(err, &certificateInvalidError) && lastBadCertificate.Swap(certificateInvalidError.Cert) != certificateInvalidError.Cert {
-			logger.Errorf(err, "Certificate error: %s", resource.ToJSON(err))
-		}
-		// The generic handler has already logged the error, no need to repeat if we don't want extra detail
-	})
+	util.AddCertificateErrorHandler(submSpec.HaltOnCertError)
 
 	restConfig, err := clientcmd.BuildConfigFromFlags(localMasterURL, localKubeconfig)
 	logger.FatalOnError(err, "Error building kubeconfig")

pkg/types/types.go

@@ -45,6 +45,7 @@ type SubmarinerSpecification struct {
 	NATEnabled                    bool
 	HealthCheckEnabled            bool `default:"true"`
 	Uninstall                     bool
+	HaltOnCertError               bool `split_words:"true"`
 	HealthCheckInterval           uint
 	HealthCheckMaxPacketLossCount uint
 	MetricsPort                   string `default:"32780"`