Serve Management gRPC and HTTP on a single 80/443 port (netbirdio#400)

This PR is a part of an effort to use standard ports (443 or 80) that are usually allowed by default in most of the environments.

Right now Management Service runs the Let'sEncrypt manager on port 443, HTTP API server on port 33071,
and a gRPC server on port 33073. There are three separate listeners.
This PR combines these listeners into one.
With this change, the HTTP and gRPC server runs on either 443 with TLS or 80 without TLS
by default (no --port specified).
Let's Encrypt manager always runs on port 443 if enabled.
The backward compatibility server runs on port 33073 (with TLS or without).
HTTP port 33071 is obsolete and not used anymore.

Newly installed agents will connect to port 443 by default instead of port 33073 if not specified otherwise.

client/internal/config.go

@@ -22,7 +22,7 @@ func ManagementURLDefault() *url.URL {
 }
 
 func init() {
-	managementURL, err := parseURL("Management URL", "https://api.wiretrustee.com:33073")
+	managementURL, err := parseURL("Management URL", "https://api.wiretrustee.com:443")
 	if err != nil {
 		panic(err)
 	}

go.mod

@@ -39,6 +39,7 @@ require (
 	github.com/rs/xid v1.3.0
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
 	github.com/stretchr/testify v1.7.1
+	golang.org/x/net v0.0.0-20220513224357-95641704303c
 	golang.org/x/term v0.0.0-20220526004731-065cf7ba2467
 )
 
@@ -96,7 +97,6 @@ require (
 	github.com/yuin/goldmark v1.4.1 // indirect
 	golang.org/x/image v0.0.0-20200430140353-33d19683fad8 // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
-	golang.org/x/net v0.0.0-20220513224357-95641704303c // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	golang.org/x/text v0.3.8-0.20211105212822-18b340fc7af2 // indirect
 	golang.org/x/tools v0.1.10 // indirect

management/cmd/management.go

@@ -1,21 +1,25 @@
 package cmd
 
 import (
-	"context"
 	"crypto/tls"
 	"errors"
 	"flag"
 	"fmt"
+	httpapi "github.com/netbirdio/netbird/management/server/http"
+	"golang.org/x/crypto/acme/autocert"
+	"golang.org/x/net/http2"
+	"golang.org/x/net/http2/h2c"
 	"io"
 	"io/fs"
 	"io/ioutil"
 	"net"
+	"net/http"
 	"os"
 	"path"
+	"strings"
 	"time"
 
 	"github.com/netbirdio/netbird/management/server"
-	"github.com/netbirdio/netbird/management/server/http"
 	"github.com/netbirdio/netbird/management/server/idp"
 	"github.com/netbirdio/netbird/util"
 
@@ -28,11 +32,16 @@ import (
 	"google.golang.org/grpc/keepalive"
 )
 
+// ManagementLegacyPort is the port that was used before by the Management gRPC server.
+// It is used for backward compatibility now.
+const ManagementLegacyPort = 33073
+
 var (
 	mgmtPort              int
 	mgmtLetsencryptDomain string
 	certFile              string
 	certKey               string
+	config                *server.Config
 
 	kaep = keepalive.EnforcementPolicy{
 		MinTime:             15 * time.Second,
@@ -46,133 +55,239 @@ var (
 		Timeout:               2 * time.Second,
 	}
 
-	// TLS enabled:
-	// 		- HTTP 80 for LetsEncrypt
-	//		- if --port not specified gRPC and HTTP servers on 443 (with multiplexing)
-	//		- if --port=X specified then run gRPC and HTTP servers on X (with multiplexing)
-	// 		- if --port=80 forbid this (throw error, otherwise we need to overcomplicate the logic with multiplexing)
-	// TLS disabled:
-	//		- if --port not specified gRPC and HTTP servers on 443 on 80 (with multiplexing)
-	//		- if --port=X specified then run gRPC and HTTP servers on 443 on X (with multiplexing)
-	// Always run gRPC on port 33073 regardless of TLS to be backward compatible
-	// Remove HTTP port 33071 from the configuration.
-
 	mgmtCmd = &cobra.Command{
 		Use:   "management",
-		Short: "start Netbird Management Server",
-		Run: func(cmd *cobra.Command, args []string) {
-			flag.Parse()
-			err := util.InitLog(logLevel, logFile)
+		Short: "start NetBird Management Server",
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			// detect whether user specified a port
+			userPort := cmd.Flag("port").Changed
+
+			var err error
+			config, err = loadMgmtConfig(mgmtConfig)
 			if err != nil {
-				log.Fatalf("failed initializing log %v", err)
+				return fmt.Errorf("failed reading provided config file: %s: %v", mgmtConfig, err)
 			}
 
-			err = handleRebrand(cmd)
+			tlsEnabled := false
+			if mgmtLetsencryptDomain != "" || (config.HttpConfig.CertFile != "" && config.HttpConfig.CertKey != "") {
+				tlsEnabled = true
+			}
+
+			if !userPort {
+				// different defaults for port when tls enabled/disabled
+				if tlsEnabled {
+					mgmtPort = 443
+				} else {
+					mgmtPort = 80
+				}
+			}
+
+			return nil
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			flag.Parse()
+			err := util.InitLog(logLevel, logFile)
 			if err != nil {
-				log.Fatalf("failed to migrate files %v", err)
+				return fmt.Errorf("failed initializing log %v", err)
 			}
 
-			config, err := loadMgmtConfig(mgmtConfig)
+			err = handleRebrand(cmd)
 			if err != nil {
-				log.Fatalf("failed reading provided config file: %s: %v", mgmtConfig, err)
+				return fmt.Errorf("failed to migrate files %v", err)
 			}
 
 			if _, err = os.Stat(config.Datadir); os.IsNotExist(err) {
 				err = os.MkdirAll(config.Datadir, os.ModeDir)
 				if err != nil {
-					log.Fatalf("failed creating datadir: %s: %v", config.Datadir, err)
+					return fmt.Errorf("failed creating datadir: %s: %v", config.Datadir, err)
 				}
 			}
 
 			store, err := server.NewStore(config.Datadir)
 			if err != nil {
-				log.Fatalf("failed creating a store: %s: %v", config.Datadir, err)
+				return fmt.Errorf("failed creating Store: %s: %v", config.Datadir, err)
 			}
 			peersUpdateManager := server.NewPeersUpdateManager()
 
 			var idpManager idp.Manager
 			if config.IdpManagerConfig != nil {
 				idpManager, err = idp.NewManager(*config.IdpManagerConfig)
 				if err != nil {
-					log.Fatalln("failed retrieving a new idp manager with err: ", err)
+					return fmt.Errorf("failed retrieving a new idp manager with err: %v", err)
 				}
 			}
 
 			accountManager, err := server.BuildManager(store, peersUpdateManager, idpManager)
 			if err != nil {
-				log.Fatalln("failed build default manager: ", err)
+				return fmt.Errorf("failed to build default manager: %v", err)
 			}
 
-			var opts []grpc.ServerOption
+			turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig)
 
-			var httpServer *http.Server
+			gRPCOpts := []grpc.ServerOption{grpc.KeepaliveEnforcementPolicy(kaep), grpc.KeepaliveParams(kasp)}
+			var certManager *autocert.Manager
+			var tlsConfig *tls.Config
+			tlsEnabled := false
 			if config.HttpConfig.LetsEncryptDomain != "" {
-				// automatically generate a new certificate with Let's Encrypt
-				certManager, err := encryption.CreateCertManager(config.Datadir, config.HttpConfig.LetsEncryptDomain)
+				certManager, err = encryption.CreateCertManager(config.Datadir, config.HttpConfig.LetsEncryptDomain)
 				if err != nil {
-					log.Fatalf("failed creating Let's Encrypt cert manager: %v", err)
+					return fmt.Errorf("failed creating LetsEncrypt cert manager: %v", err)
 				}
 				transportCredentials := credentials.NewTLS(certManager.TLSConfig())
-				opts = append(opts, grpc.Creds(transportCredentials))
-
-				httpServer = http.NewHttpsServer(config.HttpConfig, certManager, accountManager)
+				gRPCOpts = append(gRPCOpts, grpc.Creds(transportCredentials))
+				tlsEnabled = true
 			} else if config.HttpConfig.CertFile != "" && config.HttpConfig.CertKey != "" {
-				// use provided certificate
-				tlsConfig, err := loadTLSConfig(config.HttpConfig.CertFile, config.HttpConfig.CertKey)
+				tlsConfig, err = loadTLSConfig(config.HttpConfig.CertFile, config.HttpConfig.CertKey)
 				if err != nil {
-					log.Fatal("cannot load TLS credentials: ", err)
+					log.Errorf("cannot load TLS credentials: %v", err)
+					return err
 				}
 				transportCredentials := credentials.NewTLS(tlsConfig)
-				opts = append(opts, grpc.Creds(transportCredentials))
-				httpServer = http.NewHttpsServerWithTLSConfig(config.HttpConfig, tlsConfig, accountManager)
-			} else {
-				// start server without SSL
-				httpServer = http.NewHttpServer(config.HttpConfig, accountManager)
+				gRPCOpts = append(gRPCOpts, grpc.Creds(transportCredentials))
+				tlsEnabled = true
 			}
 
-			opts = append(opts, grpc.KeepaliveEnforcementPolicy(kaep), grpc.KeepaliveParams(kasp))
-			grpcServer := grpc.NewServer(opts...)
-			turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig)
-			server, err := server.NewServer(config, accountManager, peersUpdateManager, turnManager)
+			httpAPIHandler, err := httpapi.APIHandler(accountManager,
+				config.HttpConfig.AuthIssuer, config.HttpConfig.AuthAudience, config.HttpConfig.AuthKeysLocation)
 			if err != nil {
-				log.Fatalf("failed creating new server: %v", err)
+				return fmt.Errorf("failed creating HTTP API handler: %v", err)
 			}
-			mgmtProto.RegisterManagementServiceServer(grpcServer, server)
-			log.Printf("started server: localhost:%v", mgmtPort)
 
-			lis, err := net.Listen("tcp", fmt.Sprintf(":%d", mgmtPort))
+			gRPCAPIHandler := grpc.NewServer(gRPCOpts...)
+			srv, err := server.NewServer(config, accountManager, peersUpdateManager, turnManager)
 			if err != nil {
-				log.Fatalf("failed to listen: %v", err)
+				return fmt.Errorf("failed creating gRPC API handler: %v", err)
 			}
+			mgmtProto.RegisterManagementServiceServer(gRPCAPIHandler, srv)
 
-			go func() {
-				if err = grpcServer.Serve(lis); err != nil {
-					log.Fatalf("failed to serve gRpc server: %v", err)
+			var compatListener net.Listener
+			if mgmtPort != ManagementLegacyPort {
+				// The Management gRPC server was running on port 33073 previously. Old agents that are already connected to it
+				// are using port 33073. For compatibility purposes we keep running a 2nd gRPC server on port 33073.
+				compatListener, err = serveGRPC(gRPCAPIHandler, ManagementLegacyPort)
+				if err != nil {
+					return err
 				}
-			}()
+				log.Infof("running gRPC backward compatibility server: %s", compatListener.Addr().String())
+			}
 
-			go func() {
-				err = httpServer.Start()
+			rootHandler := handlerFunc(gRPCAPIHandler, httpAPIHandler)
+			var listener net.Listener
+			if certManager != nil {
+				// a call to certManager.Listener() always creates a new listener so we do it once
+				cml := certManager.Listener()
+				if mgmtPort == 443 {
+					// CertManager, HTTP and gRPC API all on the same port
+					rootHandler = certManager.HTTPHandler(rootHandler)
+					listener = cml
+				} else {
+					listener, err = tls.Listen("tcp", fmt.Sprintf(":%d", mgmtPort), certManager.TLSConfig())
+					if err != nil {
+						return fmt.Errorf("failed creating TLS listener on port %d: %v", mgmtPort, err)
+					}
+					log.Infof("running HTTP server (LetsEncrypt challenge handler): %s", cml.Addr().String())
+					serveHTTP(cml, certManager.HTTPHandler(nil))
+				}
+			} else if tlsConfig != nil {
+				listener, err = tls.Listen("tcp", fmt.Sprintf(":%d", mgmtPort), tlsConfig)
+				if err != nil {
+					return fmt.Errorf("failed creating TLS listener on port %d: %v", mgmtPort, err)
+				}
+			} else {
+				listener, err = net.Listen("tcp", fmt.Sprintf(":%d", mgmtPort))
 				if err != nil {
-					log.Fatalf("failed to serve http server: %v", err)
+					return fmt.Errorf("failed creating TCP listener on port %d: %v", mgmtPort, err)
 				}
-			}()
+			}
+
+			log.Infof("running HTTP server and gRPC server on the same port: %s", listener.Addr().String())
+			serveGRPCWithHTTP(listener, rootHandler, tlsEnabled)
 
 			SetupCloseHandler()
+
 			<-stopCh
-			log.Println("Receive signal to stop running Management server")
-			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-			defer cancel()
-			err = httpServer.Stop(ctx)
-			if err != nil {
-				log.Fatalf("failed stopping the http server %v", err)
+			_ = listener.Close()
+			if certManager != nil {
+				_ = certManager.Listener().Close()
 			}
+			gRPCAPIHandler.Stop()
+			log.Infof("stopped Management Service")
 
-			grpcServer.Stop()
+			return nil
 		},
 	}
 )
 
+func notifyStop(msg string) {
+	select {
+	case stopCh <- 1:
+		log.Error(msg)
+	default:
+		// stop has been already called, nothing to report
+	}
+}
+
+func serveGRPC(grpcServer *grpc.Server, port int) (net.Listener, error) {
+	listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
+	if err != nil {
+		return nil, err
+	}
+	go func() {
+		err := grpcServer.Serve(listener)
+		if err != nil {
+			notifyStop(fmt.Sprintf("failed running gRPC server on port %d: %v", port, err))
+		}
+	}()
+	return listener, nil
+}
+
+func serveHTTP(httpListener net.Listener, handler http.Handler) {
+	go func() {
+		err := http.Serve(httpListener, handler)
+		if err != nil {
+			notifyStop(fmt.Sprintf("failed running HTTP server: %v", err))
+		}
+	}()
+}
+
+func serveGRPCWithHTTP(listener net.Listener, handler http.Handler, tlsEnabled bool) {
+	go func() {
+		var err error
+		if tlsEnabled {
+			err = http.Serve(listener, handler)
+		} else {
+			// the following magic is needed to support HTTP2 without TLS
+			// and still share a single port between gRPC and HTTP APIs
+			h1s := &http.Server{
+				Handler: h2c.NewHandler(handler, &http2.Server{}),
+			}
+			err = h1s.Serve(listener)
+		}
+
+		if err != nil {
+			select {
+			case stopCh <- 1:
+				log.Errorf("failed to serve HTTP and gRPC server: %v", err)
+			default:
+				// stop has been already called, nothing to report
+			}
+		}
+	}()
+}
+
+func handlerFunc(gRPCHandler *grpc.Server, httpHandler http.Handler) http.Handler {
+	return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+		grpcHeader := strings.HasPrefix(request.Header.Get("Content-Type"), "application/grpc") ||
+			strings.HasPrefix(request.Header.Get("Content-Type"), "application/grpc+proto")
+		fmt.Println(grpcHeader)
+		if request.ProtoMajor == 2 && grpcHeader {
+			gRPCHandler.ServeHTTP(writer, request)
+		} else {
+			httpHandler.ServeHTTP(writer, request)
+		}
+	})
+}
+
 func loadMgmtConfig(mgmtConfigPath string) (*server.Config, error) {
 	config := &server.Config{}
 	_, err := util.ReadJson(mgmtConfigPath, config)

management/cmd/root.go

@@ -60,7 +60,7 @@ func init() {
 	oldDefaultMgmtConfig = oldDefaultMgmtConfigDir + "/management.json"
 	oldDefaultLogFile = oldDefaultLogDir + "/management.log"
 
-	mgmtCmd.Flags().IntVar(&mgmtPort, "port", 33073, "server port to listen on")
+	mgmtCmd.Flags().IntVar(&mgmtPort, "port", 80, "server port to listen on (defaults to 443 if TLS is enabled, 80 otherwise")
 	mgmtCmd.Flags().StringVar(&mgmtDataDir, "datadir", defaultMgmtDataDir, "server data directory location")
 	mgmtCmd.Flags().StringVar(&mgmtConfig, "config", defaultMgmtConfig, "Netbird config file location. Config params specified via command line (e.g. datadir) have a precedence over configuration from this file")
 	mgmtCmd.Flags().StringVar(&mgmtLetsencryptDomain, "letsencrypt-domain", "", "a domain to issue Let's Encrypt certificate for. Enables TLS using Let's Encrypt. Will fetch and renew certificate, and run the server with TLS")

management/server/config.go

@@ -49,7 +49,6 @@ type HttpServerConfig struct {
 	CertFile string
 	//CertKey is the location of the certificate private key
 	CertKey string
-	Address string
 	// AuthAudience identifies the recipients that the JWT is intended for (aud in JWT)
 	AuthAudience string
 	// AuthIssuer identifies principal that issued the JWT.