Allow halting on certificate errors

When certificate errors are encountered, the fix is usually to restart
the affected pod. To allow this to happen automatically, add a
configuration setting for the gateway agent; extend the mechanism to
the route agent.

The setting is disabled by default; it will be enabled by default by
the operator.

Signed-off-by: Stephen Kitt <skitt@redhat.com>

main.go

@@ -20,7 +20,6 @@ package main
 
 import (
 	"context"
-	"crypto/x509"
 	"errors"
 	"flag"
 	"net/http"
@@ -33,8 +32,8 @@ import (
 	"github.com/submariner-io/admiral/pkg/log"
 	"github.com/submariner-io/admiral/pkg/log/kzerolog"
 	"github.com/submariner-io/admiral/pkg/names"
-	"github.com/submariner-io/admiral/pkg/resource"
 	"github.com/submariner-io/admiral/pkg/syncer/broker"
+	"github.com/submariner-io/admiral/pkg/util"
 	admversion "github.com/submariner-io/admiral/pkg/version"
 	"github.com/submariner-io/admiral/pkg/watcher"
 	subv1 "github.com/submariner-io/submariner/pkg/apis/submariner.io/v1"
@@ -44,7 +43,6 @@ import (
 	"github.com/submariner-io/submariner/pkg/natdiscovery"
 	"github.com/submariner-io/submariner/pkg/types"
 	"github.com/submariner-io/submariner/pkg/versions"
-	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
@@ -105,18 +103,7 @@ func main() {
 
 	var err error
 
-	//nolint:reassign // We need to reassign ErrorHandlers to register our handler
-	utilruntime.ErrorHandlers = append(utilruntime.ErrorHandlers, func(err error) {
-		var unknownAuthorityError x509.UnknownAuthorityError
-		if errors.As(err, &unknownAuthorityError) && lastBadCertificate.Swap(unknownAuthorityError.Cert) != unknownAuthorityError.Cert {
-			logger.Errorf(err, "Certificate error: %s", resource.ToJSON(err))
-		}
-		var certificateInvalidError x509.CertificateInvalidError
-		if errors.As(err, &certificateInvalidError) && lastBadCertificate.Swap(certificateInvalidError.Cert) != certificateInvalidError.Cert {
-			logger.Errorf(err, "Certificate error: %s", resource.ToJSON(err))
-		}
-		// The generic handler has already logged the error, no need to repeat if we don't want extra detail
-	})
+	util.AddCertificateErrorHandler(submSpec.HaltOnCertificateError)
 
 	restConfig, err := clientcmd.BuildConfigFromFlags(localMasterURL, localKubeconfig)
 	logger.FatalOnError(err, "Error building kubeconfig")

pkg/routeagent_driver/environment/env.go

@@ -19,11 +19,12 @@ limitations under the License.
 package environment
 
 type Specification struct {
-	ClusterID   string
-	Namespace   string
-	ClusterCidr []string
-	ServiceCidr []string
-	GlobalCidr  []string
-	Uninstall   bool
-	WaitForNode bool
+	ClusterID              string
+	Namespace              string
+	ClusterCidr            []string
+	ServiceCidr            []string
+	GlobalCidr             []string
+	Uninstall              bool
+	WaitForNode            bool
+	HaltOnCertificateError bool
 }

pkg/routeagent_driver/main.go

@@ -24,13 +24,15 @@ import (
 	"io/fs"
 	"os"
 	"strconv"
+	"sync/atomic"
 	"time"
 
 	"github.com/kelseyhightower/envconfig"
 	"github.com/pkg/errors"
 	"github.com/submariner-io/admiral/pkg/log"
 	"github.com/submariner-io/admiral/pkg/log/kzerolog"
 	"github.com/submariner-io/admiral/pkg/names"
+	"github.com/submariner-io/admiral/pkg/util"
 	admversion "github.com/submariner-io/admiral/pkg/version"
 	"github.com/submariner-io/admiral/pkg/watcher"
 	v1 "github.com/submariner-io/submariner/pkg/apis/submariner.io/v1"
@@ -60,10 +62,11 @@ import (
 )
 
 var (
-	masterURL   string
-	kubeconfig  string
-	logger      = log.Logger{Logger: logf.Log.WithName("main")}
-	showVersion = false
+	masterURL          string
+	kubeconfig         string
+	logger             = log.Logger{Logger: logf.Log.WithName("main")}
+	showVersion        = false
+	lastBadCertificate atomic.Value
 )
 
 func main() {
@@ -152,6 +155,8 @@ func main() {
 		return
 	}
 
+	util.AddCertificateErrorHandler(env.HaltOnCertificateError)
+
 	if err = annotateNode(env.ClusterCidr, k8sClientSet); err != nil {
 		logger.Errorf(err, "Error while annotating the node")
 	}

pkg/types/types.go

@@ -45,6 +45,7 @@ type SubmarinerSpecification struct {
 	NATEnabled                    bool
 	HealthCheckEnabled            bool `default:"true"`
 	Uninstall                     bool
+	HaltOnCertificateError        bool
 	HealthCheckInterval           uint
 	HealthCheckMaxPacketLossCount uint
 	MetricsPort                   string `default:"32780"`