Add unit tests

RemoteCluster Tests
-------------------

Test that member announce is written and connectivity is set
to true.

TODO: Find out if fakeClient can error writes so we can test
disconnection case

NOTE: used mockgen to generate mocks for Manager from controller-runtime
library and for RemoteClusterManager. TODO: Add mockgen to Makefile
so it always gets generated automatically.

RemoteClusterManager and LeaderElector tests
--------------------------------------------

Test adding a new cluster and leader election

Test leader election when elected leader becomes disconnected

TODO: test removing a cluster

NOTE: Used pegomock to generate a mock for RemoteCluster so that
we can verify StartMonitoring is invoked. TODO: Use mockgen instead
but for now couldnt get it working due to RemoteCluster embedding
CommonArea interface (Ref:golang/mock#85)
Also had to edit the mock file generated by pegomock due to go test
complaining of a race in mock code. Added a mutex lock in the mock
code although ideally we should not edit generated mock files

Signed-off-by: Aravinda Kidambi <akidambi@vmware.com>

go.mod

@@ -20,6 +20,7 @@ require (
 	github.com/containernetworking/plugins v0.8.7
 	github.com/coreos/go-iptables v0.6.0
 	github.com/elazarl/goproxy v0.0.0-20190911111923-ecfe977594f1 // indirect
+	github.com/go-logr/logr v0.4.0
 	github.com/go-openapi/spec v0.19.5
 	github.com/gogo/protobuf v1.3.2
 	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e
@@ -33,6 +34,7 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822
 	github.com/onsi/ginkgo v1.16.4
 	github.com/onsi/gomega v1.13.0
+	github.com/petergtz/pegomock v2.9.0+incompatible
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.11.0
 	github.com/prometheus/common v0.26.0
@@ -46,6 +48,7 @@ require (
 	github.com/ti-mo/conntrack v0.4.0
 	github.com/vishvananda/netlink v1.1.1-0.20210510164352-d17758a128bf
 	github.com/vmware/go-ipfix v0.5.10
+	go.uber.org/multierr v1.6.0
 	golang.org/x/crypto v0.0.0-20210503195802-e9a32991a82e
 	golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6
 	golang.org/x/mod v0.4.2
@@ -100,7 +103,6 @@ require (
 	github.com/fatih/color v1.10.0 // indirect
 	github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
-	github.com/go-logr/logr v0.4.0 // indirect
 	github.com/go-logr/zapr v0.4.0 // indirect
 	github.com/go-openapi/jsonpointer v0.19.3 // indirect
 	github.com/go-openapi/jsonreference v0.19.3 // indirect
@@ -147,11 +149,11 @@ require (
 	go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489 // indirect
 	go.opencensus.io v0.22.3 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
-	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.17.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
 	golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect
 	golang.org/x/text v0.3.6 // indirect
+	golang.org/x/tools v0.1.1 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	golang.zx2c4.com/wireguard v0.0.0-20210427022245-097af6e1351b // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect

go.sum

@@ -588,6 +588,8 @@ github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
+github.com/petergtz/pegomock v2.9.0+incompatible h1:BKfb5XfkJfehe5T+O1xD4Zm26Sb9dnRj7tHxLYwUPiI=
+github.com/petergtz/pegomock v2.9.0+incompatible/go.mod h1:nuBLWZpVyv/fLo56qTwt/AUau7jgouO1h7bEvZCq82o=
 github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pion/dtls/v2 v2.0.3 h1:3qQ0s4+TXD00rsllL8g8KQcxAs+Y/Z6oz618RXX6p14=
 github.com/pion/dtls/v2 v2.0.3/go.mod h1:TUjyL8bf8LH95h81Xj7kATmzMRt29F/4lxpIPj2Xe4Y=

multicluster/controllers/multicluster/core/leader_elector.go

@@ -73,8 +73,10 @@ func (m *remoteClusterManager) StartLeaderElection() (context.CancelFunc, error)
 				klog.Infof("Stopping leader election")
 				return
 			case <-ticker.C:
+				mutex.Lock()
 				// 5 second timer has gone off, check if election must be done
 				if !m.needElection {
+					klog.Info("Check if leader election is necessary")
 					if m.electedLeaderCluster != nil && !m.electedLeaderCluster.IsConnected() {
 						klog.Infof("Leader disconnected", "leader", m.electedLeaderCluster.GetClusterID())
 						m.electedLeaderCluster = nil
@@ -83,14 +85,17 @@ func (m *remoteClusterManager) StartLeaderElection() (context.CancelFunc, error)
 						// do we have any member that is connected?
 						for _, cluster := range m.remoteClusters {
 							if cluster.IsConnected() {
+								klog.Info("Have atleast one connected leader")
 								m.needElection = true
 							}
 						}
 					}
 				}
 				if m.needElection {
+					klog.Info("Perform leader election")
 					m.DoLeaderElection()
 				}
+				mutex.Unlock()
 			}
 		}
 	}()
@@ -118,8 +123,8 @@ func (m *remoteClusterManager) DoLeaderElection() {
 		electedLeaderClusterID := connectedClusterIDs[electedLeaderIndex]
 		m.electedLeaderCluster = m.remoteClusters[electedLeaderClusterID]
 		m.needElection = false
-		m.electedLeaderCluster.StartMonitoring()
 		klog.Infof("Election completed", "leader", electedLeaderClusterID)
+		m.electedLeaderCluster.StartMonitoring()
 		return
 	}
 	// couldnt elect a leader, will try next round