loxilb-iogh-877 Initial support for egress

api/loxinlp/nlp.go

@@ -1271,6 +1271,26 @@ func AddRouteNoHook(DestinationIPNet, gateway, proto string) int {
 	return ret
 }
 
+func GetRouteNoHook(destination string) ([]string, error) {
+	var gws []string
+
+	dst := net.ParseIP(destination)
+	if dst == nil {
+		return []string{}, errors.New("invalid destination")
+	}
+
+	rts, err := nlp.RouteGet(dst)
+	if err != nil {
+		return []string{}, errors.New("invalid rt destination")
+	}
+
+	for _, rt := range rts {
+		gws = append(gws, rt.Gw.String())
+	}
+
+	return gws, nil
+}
+
 func DelRouteNoHook(DestinationIPNet string) int {
 	var ret int
 	var route nlp.Route

api/restapi/handler/firewall.go

@@ -59,6 +59,7 @@ func ConfigPostFW(params operations.PostConfigFirewallParams) middleware.Respond
 	Opts.DoSnat = params.Attr.Opts.DoSnat
 	Opts.ToIP = params.Attr.Opts.ToIP
 	Opts.ToPort = uint16(params.Attr.Opts.ToPort)
+	Opts.OnDefault = params.Attr.Opts.OnDefault
 
 	FW.Rule = Rules
 	FW.Opts = Opts
@@ -179,6 +180,7 @@ func ConfigGetFW(params operations.GetConfigFirewallAllParams) middleware.Respon
 		tmpOpts.DoSnat = FW.Opts.DoSnat
 		tmpOpts.ToIP = FW.Opts.ToIP
 		tmpOpts.ToPort = int64(FW.Opts.ToPort)
+		tmpOpts.OnDefault = FW.Opts.OnDefault
 		tmpOpts.Counter = FW.Opts.Counter
 		tmpResult.RuleArguments = &tmpRule
 		tmpResult.Opts = &tmpOpts

api/swagger.yml

@@ -3781,6 +3781,9 @@ definitions:
       toPort:
         type: integer
         description: Modify to given Port (Zero if port is not to be modified)
+      onDefault:
+        type: boolean
+        description: Trigger only on default cases
       counter:
         type: string
         description: traffic counters

common/common.go

@@ -408,6 +408,8 @@ type FwOptArg struct {
 	DoSnat bool   `json:"doSnat"`
 	ToIP   string `json:"toIP"`
 	ToPort uint16 `json:"toPort"`
+	// OnDefault - Trigger only on default cases
+	OnDefault bool `json:"onDefault"`
 	// Counter - Traffic counter
 	Counter string `json:"counter"`
 }

options/options.go

@@ -40,4 +40,5 @@ var Opts struct {
 	ConfigPath        string         `long:"config-path" description:"Config file path" default:"/etc/loxilb/"`
 	ProxyModeOnly     bool           `long:"proxyonlymode" description:"Run loxilb in proxy mode only, no Datapath"`
 	WhiteList         string         `long:"whitelist" description:"Regex string of whitelisted interface(experimental)" default:"none"`
+	ClusterInterface  string         `long:"clusterinterface" description:"cluster interface for egress HA" default:""`
 }

pkg/loxinet/cluster.go

@@ -80,7 +80,11 @@ type CIStateH struct {
 	Bs          *bfd.Struct
 	ClusterNet  string
 	ClusterNet6 string
+	ClusterGw   string
+	ClusterGw6  string
 	ClusterIf   string
+	OGw         []string
+	OGw6        []string
 }
 
 func (ci *CIStateH) BFDSessionNotify(instance string, remote string, ciState string) {
@@ -140,6 +144,16 @@ func (ci *CIStateH) CISpawn() {
 	}
 }
 
+// CIStateGetInst - routine to get HA state
+func (h *CIStateH) CIStateGetInst(inst string) (string, error) {
+
+	if ci, ok := h.ClusterMap[inst]; ok {
+		return ci.StateStr, nil
+	}
+
+	return "NOT_DEFINED", errors.New("not found")
+}
+
 // CIInit - routine to initialize Cluster context
 func CIInit(args CIKAArgs) *CIStateH {
 	var nCIh = new(CIStateH)
@@ -168,9 +182,8 @@ func CIInit(args CIKAArgs) *CIStateH {
 
 	nCIh.NodeMap = make(map[string]*ClusterNode)
 
-	args.CDev = "eth0"
 	if args.CDev != "" {
-		tk.LogIt(tk.LogError, "cluster-dev name\n")
+		tk.LogIt(tk.LogInfo, "cluster-dev name %s\n", args.CDev)
 		_, err := net.InterfaceByName(args.CDev)
 		if err != nil {
 			tk.LogIt(tk.LogError, "cluster-dev name error\n")
@@ -225,24 +238,40 @@ func CIInit(args CIKAArgs) *CIStateH {
 			return nil
 		}
 
+		nlp.DelAddrNoHook(ip.String()+"/16", clusterIfName)
 		if nlp.AddAddrNoHook(ip.String()+"/16", clusterIfName) < 0 {
 			tk.LogIt(tk.LogError, "Failed to add Cluster Addr %s:%s\n", ip.String(), clusterIfName)
 			return nil
 		}
 
+		nlp.DelAddrNoHook(ip6.String()+"/16", clusterIfName)
 		if nlp.AddAddrNoHook(ip6.String()+"/64", clusterIfName) < 0 {
 			tk.LogIt(tk.LogError, "Failed to add Cluster Addr %s:%s\n", ip6.String(), clusterIfName)
 			nCIh.ClusterNet6 = ""
 		} else {
 			nCIh.ClusterNet6 = clusterCIDR6
 		}
 
-		tk.LogIt(tk.LogInfo, "Cluster IP address %s\n", ip.String())
-		tk.LogIt(tk.LogInfo, "Cluster IP6 address %s\n", ip6.String())
+		gw := make(net.IP, len(ip))
+		copy(gw, ip)
+		gw[len(gw)-1] = 254
+
+		gw6 := make(net.IP, len(ip6))
+		copy(gw6, ip6)
+		gw6[len(gw6)-1] = 254
 
 		nCIh.ClusterIf = args.CDev
 		nCIh.ClusterNet = clusterCIDR
 		nCIh.ClusterNet6 = clusterCIDR6
+		nCIh.ClusterGw = gw.String()
+		nCIh.ClusterGw6 = gw6.String()
+
+		nCIh.OGw, _ = nlp.GetRouteNoHook("8.8.8.8")
+		nCIh.OGw6, _ = nlp.GetRouteNoHook("2001:4860:4860::8888")
+
+		tk.LogIt(tk.LogInfo, "Cluster IP address %s GW %s oGW %v\n", ip.String(), nCIh.ClusterGw, nCIh.OGw)
+		tk.LogIt(tk.LogInfo, "Cluster IP6 address %s GW6 %s oGw6 %v\n", ip6.String(), nCIh.ClusterGw, nCIh.OGw6)
+
 	}
 
 	return nCIh
@@ -293,9 +322,6 @@ func (ci *CIStateH) CIDestroy() {
 		ip6[len(ip)-2] = ifIP6[len(ifIP6)-2]
 		ip6[len(ip)-1] = ifIP6[len(ifIP6)-1]
 
-		tk.LogIt(tk.LogInfo, "Cluster IP address %s deleted\n", ip.String())
-		tk.LogIt(tk.LogInfo, "Cluster IP6 address %s deleted\n", ip6.String())
-
 		clusterIfName := fmt.Sprintf("vxlan%d", ClusterNetID)
 
 		if nlp.DelAddrNoHook(ip.String()+"/16", clusterIfName) < 0 {
@@ -306,20 +332,93 @@ func (ci *CIStateH) CIDestroy() {
 			tk.LogIt(tk.LogError, "Failed to delete Cluster Addr %s:%s\n", ip6.String(), clusterIfName)
 		}
 
+		tk.LogIt(tk.LogInfo, "Cluster IP address %s deleted\n", ip.String())
+		tk.LogIt(tk.LogInfo, "Cluster IP6 address %s deleted\n", ip6.String())
+
 		if nlp.DelVxLANNoHook(ClusterNetID) < 0 {
 			tk.LogIt(tk.LogError, "Failed to delete Cluster Network\n")
 		}
+
+		if len(mh.has.OGw) > 0 {
+			nlp.DelRouteNoHook("0.0.0.0/0")
+			for _, gw := range mh.has.OGw {
+				nlp.AddRouteNoHook("0.0.0.0/0", gw, "static")
+			}
+		}
+		if len(mh.has.OGw6) > 0 {
+			nlp.DelRouteNoHook("::/0")
+			for _, gw := range mh.has.OGw6 {
+				nlp.AddRouteNoHook("::/0", gw, "static")
+			}
+		}
 	}
 }
 
-// CIStateGetInst - routine to get HA state
-func (h *CIStateH) CIStateGetInst(inst string) (string, error) {
-
-	if ci, ok := h.ClusterMap[inst]; ok {
-		return ci.StateStr, nil
+// CIAddClusterRoute - routine to add a cluster route
+func (h *CIStateH) CIAddClusterRoute(dest string, add bool) {
+
+	if add {
+		found := false
+		if tk.IsNetIPv4(dest) {
+			gws, _ := nlp.GetRouteNoHook("8.8.8.8")
+			for _, gw := range gws {
+				if gw == dest {
+					found = true
+					break
+				}
+			}
+			fmt.Printf("gws = %v: dest %s\n", gws, dest)
+			if !found {
+				nlp.DelRouteNoHook("0.0.0.0/0")
+				nlp.AddRouteNoHook("0.0.0.0/0", dest, "static")
+			}
+		} else {
+			found = false
+			gws, _ := nlp.GetRouteNoHook("2001:4860:4860::8888")
+			for _, gw := range gws {
+				if gw == dest {
+					found = true
+					break
+				}
+			}
+			if !found {
+				nlp.DelRouteNoHook("::/0")
+				nlp.AddRouteNoHook("::/0", dest, "static")
+			}
+		}
+	} else {
+		found := false
+		if tk.IsNetIPv4(dest) {
+			gws, _ := nlp.GetRouteNoHook("8.8.8.8")
+			for _, gw := range gws {
+				if gw == dest {
+					found = true
+					break
+				}
+			}
+			if found {
+				nlp.DelRouteNoHook("0.0.0.0/0")
+				for _, gw := range mh.has.OGw {
+					nlp.AddRouteNoHook("0.0.0.0/0", gw, "static")
+				}
+			}
+		} else {
+			found = false
+			gws, _ := nlp.GetRouteNoHook("2001:4860:4860::8888")
+			for _, gw := range gws {
+				if gw == dest {
+					found = true
+					break
+				}
+			}
+			if found {
+				nlp.DelRouteNoHook("::/0")
+				for _, gw := range mh.has.OGw {
+					nlp.AddRouteNoHook("::", gw, "static")
+				}
+			}
+		}
 	}
-
-	return "NOT_DEFINED", errors.New("not found")
 }
 
 // CIStateGet - routine to get HA state
@@ -381,7 +480,7 @@ func (h *CIStateH) CIStateUpdate(cm cmn.HASMod) (int, error) {
 		if mh.bgp != nil {
 			mh.bgp.UpdateCIState(cm.Instance, ci.State, ci.Vip)
 		}
-		go mh.zr.Rules.RuleVIPSyncToClusterState()
+		go mh.zr.Rules.RulesSyncToClusterState()
 		return ci.State, nil
 	}
 

pkg/loxinet/dpbroker.go

@@ -239,6 +239,7 @@ type FwDpWorkQ struct {
 	FwVal1   uint16
 	FwVal2   uint32
 	FwRecord bool
+	OnDflt   bool
 }
 
 // NatT - type of NAT

pkg/loxinet/dpebpf_linux.go

@@ -896,7 +896,11 @@ func DpRouteMod(w *RouteDpWorkQ) int {
 		C.memset(unsafe.Pointer(dat), 0, C.sizeof_struct_dp_rt_tact)
 
 		if w.NMax > 0 {
-			dat.ca.act_type = C.DP_SET_RT_NHNUM
+			if w.Dst.IP.IsUnspecified() {
+				dat.ca.act_type = C.DP_SET_RT_NHNUM_DFLT
+			} else {
+				dat.ca.act_type = C.DP_SET_RT_NHNUM
+			}
 			act = (*rtL3NhAct)(getPtrOffset(unsafe.Pointer(dat),
 				C.sizeof_struct_dp_cmn_act))
 			act.naps = C.ushort(w.NMax)

pkg/loxinet/layer3.go

@@ -557,6 +557,14 @@ func (ifa *Ifa) DP(work DpWorkT) int {
 
 	rmWq.PortNum = port.PortNo
 
+	if port.SInfo.PortType&cmn.PortVxlanBr == cmn.PortVxlanBr {
+		if port.SInfo.PortReal == nil {
+			tk.LogIt(tk.LogError, "No real port : %s\n", port.Name)
+			ifa.Sync = DpCreateErr
+			return -1
+		}
+	}
+
 	mh.dp.ToDpCh <- rmWq
 
 	if port.SInfo.PortType&cmn.PortVxlanBr == cmn.PortVxlanBr {
@@ -565,7 +573,9 @@ func (ifa *Ifa) DP(work DpWorkT) int {
 		rmWq.Status = &ifa.Sync
 
 		if port.SInfo.PortReal == nil {
-			return 0
+			tk.LogIt(tk.LogError, "No real port : %s(error)\n", port.Name)
+			ifa.Sync = DpCreateErr
+			return -1
 		}
 
 		up := port.SInfo.PortReal

pkg/loxinet/loxinet.go

@@ -216,7 +216,12 @@ func sysctlInit() {
 func loxiNetInit() {
 	var rpcMode int
 
-	kaArgs := KAString2Mode(opts.Opts.Ka)
+	// Initialize logger and specify the log file
+	logfile := fmt.Sprintf("%s%s.log", "/var/log/loxilb", os.Getenv("HOSTNAME"))
+	logLevel := LogString2Level(opts.Opts.LogLevel)
+	mh.logger = tk.LogItInit(logfile, logLevel, true)
+
+	kaArgs := KAString2Mode(opts.Opts.Ka, opts.Opts.ClusterInterface)
 	clusterMode := false
 	if opts.Opts.ClusterNodes != "none" {
 		clusterMode = true
@@ -228,11 +233,6 @@ func loxiNetInit() {
 		os.Exit(1)
 	}
 
-	// Initialize logger and specify the log file
-	logfile := fmt.Sprintf("%s%s.log", "/var/log/loxilb", os.Getenv("HOSTNAME"))
-	logLevel := LogString2Level(opts.Opts.LogLevel)
-	mh.logger = tk.LogItInit(logfile, logLevel, true)
-
 	// It is important to make sure loxilb's eBPF filesystem
 	// is in place and mounted to make sure maps are pinned properly
 	if !opts.Opts.ProxyModeOnly {

pkg/loxinet/port.go

@@ -806,6 +806,9 @@ func (P *PortsH) PortTicker() {
 				tk.LogIt(tk.LogError, "port - %s set ovl-port(%s)\n", rp.Name, port.Name)
 				rp.SInfo.PortOvl = port
 			}
+			if port.SInfo.PortReal != rp {
+				port.SInfo.PortReal = rp
+			}
 		}
 
 		continue