Parse application interface order fields

Parse newly added interface order fields and the EnforceNetworkInterfaceOrder
boolean option.

Signed-off-by: Milan Lenco <milan@zededa.com>

pkg/pillar/cmd/zedagent/parseconfig.go

@@ -689,6 +689,8 @@ func parseAppInstanceConfig(getconfigCtx *getconfigContext,
 		appInstance.FixedResources.EnableVncShimVM = cfgApp.Fixedresources.EnableVncShimVm
 		appInstance.FixedResources.VncDisplay = cfgApp.Fixedresources.VncDisplay
 		appInstance.FixedResources.VncPasswd = cfgApp.Fixedresources.VncPasswd
+		appInstance.FixedResources.EnforceNetworkInterfaceOrder =
+			cfgApp.Fixedresources.EnforceNetworkInterfaceOrder
 		appInstance.DisableLogs = cfgApp.Fixedresources.DisableLogs
 		appInstance.MetaDataType = types.MetaDataType(cfgApp.MetaDataType)
 		appInstance.Delay = time.Duration(cfgApp.StartDelayInSeconds) * time.Second
@@ -734,6 +736,9 @@ func parseAppInstanceConfig(getconfigCtx *getconfigContext,
 			} else if ioa.Type == types.IoCAN || ioa.Type == types.IoVCAN || ioa.Type == types.IoLCAN {
 				log.Functionf("Got CAN adapter")
 			}
+			if ioa.Type.IsNet() && appInstance.FixedResources.EnforceNetworkInterfaceOrder {
+				ioa.IntfOrder = adapter.GetInterfaceOrder()
+			}
 			appInstance.IoAdapterList = append(appInstance.IoAdapterList, ioa)
 		}
 		log.Functionf("Got adapters %v", appInstance.IoAdapterList)
@@ -2439,11 +2444,12 @@ func parseAppNetAdapterConfig(appInstance *types.AppInstanceConfig,
 				intfEnt.Name, adapterCfg.Error)
 		}
 	}
-	// sort based on intfOrder
-	// XXX remove? Debug?
-	if len(appInstance.AppNetAdapterList) > 1 {
-		log.Functionf("XXX pre sort %+v", appInstance.AppNetAdapterList)
-	}
+
+	// Sort based on IntfOrder. When EnforceNetworkInterfaceOrder is enabled, this is done
+	// only for troubleshooting purposes to make the pubsub messages containing application
+	// interface list easier to read. Interface order is still determined by the IntfOrder
+	// attribute, and that includes direct attachments, not based on the order of items
+	// inside AppNetAdapterList.
 	sort.Slice(appInstance.AppNetAdapterList[:],
 		func(i, j int) bool {
 			return appInstance.AppNetAdapterList[i].IntfOrder <
@@ -2490,8 +2496,6 @@ func parseAppNetAdapterConfigEntry(
 
 	adapterCfg := new(types.AppNetAdapterConfig)
 	adapterCfg.Name = intfEnt.Name
-	// XXX set adapterCfg.IntfOrder from API once available
-	var intfOrder int32
 	// Lookup NetworkInstance ID
 	networkInstanceEntry := lookupNetworkInstanceId(intfEnt.NetworkId,
 		cfgNetworkInstances)
@@ -2553,6 +2557,7 @@ func parseAppNetAdapterConfigEntry(
 		}
 	}
 
+	var aclIntfOrder uint32
 	adapterCfg.ACLs = make([]types.ACE, len(intfEnt.Acls))
 	for aclIdx, acl := range intfEnt.Acls {
 		aclCfg := new(types.ACE)
@@ -2561,9 +2566,11 @@ func parseAppNetAdapterConfigEntry(
 		aclCfg.Actions = make([]types.ACEAction,
 			len(acl.Actions))
 		aclCfg.RuleID = acl.Id
-		// XXX temporary until we get an intfOrder in the API
-		if intfOrder == 0 {
-			intfOrder = acl.Id
+		// When EnforceNetworkInterfaceOrder is disabled, we fall back to the previous
+		// interface ordering method, where virtual interfaces are ordered according to ACL
+		// IDs and direct attachments come after virtual interfaces.
+		if aclIntfOrder == 0 && acl.Id > 0 {
+			aclIntfOrder = uint32(acl.Id)
 		}
 		aclCfg.Name = acl.Name
 		aclCfg.Dir = types.ACEDirection(acl.Dir)
@@ -2587,8 +2594,11 @@ func parseAppNetAdapterConfigEntry(
 		}
 		adapterCfg.ACLs[aclIdx] = *aclCfg
 	}
-	// XXX set adapterCfg.IntfOrder from API once available
-	adapterCfg.IntfOrder = intfOrder
+	if cfgApp.Fixedresources.EnforceNetworkInterfaceOrder {
+		adapterCfg.IntfOrder = intfEnt.GetInterfaceOrder()
+	} else {
+		adapterCfg.IntfOrder = aclIntfOrder
+	}
 	adapterCfg.AccessVlanID = intfEnt.AccessVlanId
 	adapterCfg.AllowToDiscover = intfEnt.AllowToDiscover
 
@@ -2822,6 +2832,37 @@ func checkAndPublishAppInstanceConfig(getconfigCtx *getconfigContext,
 		config.Errors = append(config.Errors, err.Error())
 	}
 
+	// If EnforceNetworkInterfaceOrder is enabled, check that every network interface
+	// has unique order number.
+	if config.FixedResources.EnforceNetworkInterfaceOrder {
+		intfOrderMap := make(map[uint32]string)
+		for _, adapter := range config.AppNetAdapterList {
+			if adapter2, duplicate := intfOrderMap[adapter.IntfOrder]; duplicate {
+				err := fmt.Errorf("virtual network adapter %s has the same interface "+
+					"order (%d) configured as adapter %s", adapter.Name, adapter.IntfOrder,
+					adapter2)
+				log.Error(err)
+				config.Errors = append(config.Errors, err.Error())
+				continue
+			}
+			intfOrderMap[adapter.IntfOrder] = adapter.Name
+		}
+		for _, adapter := range config.IoAdapterList {
+			if !adapter.Type.IsNet() {
+				continue
+			}
+			if adapter2, duplicate := intfOrderMap[adapter.IntfOrder]; duplicate {
+				err := fmt.Errorf("directly attached network adapter %s has the same "+
+					"interface order (%d) configured as adapter %s", adapter.Name,
+					adapter.IntfOrder, adapter2)
+				log.Error(err)
+				config.Errors = append(config.Errors, err.Error())
+				continue
+			}
+			intfOrderMap[adapter.IntfOrder] = adapter.Name
+		}
+	}
+
 	pub.Publish(key, config)
 }
 

pkg/pillar/cmd/zedagent/reportinfo.go

@@ -644,8 +644,10 @@ func PublishDeviceInfoToZedCloud(ctx *zedagentContext, dest destinationBitset) {
 	// TODO: Enhance capability reporting with a bitmap-like approach for increased granularity.
 	// We report the snapshot capability despite the fact that we support snapshots only
 	// for file-based volumes. If a controller tries to make a snapshot of ZFS-based volume
-	// device returns a runtime error.
-	ReportDeviceInfo.ApiCapability = info.APICapability_API_CAPABILITY_ADAPTER_USER_LABELS
+	// device returns a runtime error. Similarly, we only support enforced application network
+	// interface order for the KVM hypervisor. If enabled for application deployed under Xen
+	// or Kubevirt hypervisor, EVE returns error and the application will not be started.
+	ReportDeviceInfo.ApiCapability = info.APICapability_API_CAPABILITY_ENFORCED_NET_INTERFACE_ORDER
 
 	// Report if there is a local override of profile
 	if ctx.getconfigCtx.sideController.currentProfile !=

pkg/pillar/cmd/zedmanager/zedmanager.go

@@ -1095,7 +1095,7 @@ func handleCreate(ctxArg interface{}, key string,
 	if len(config.Errors) > 0 {
 		// Combine all errors from Config parsing state and send them in Status
 		for i, errStr := range config.Errors {
-			allErrors += errStr
+			allErrors += errStr + "\n"
 			log.Errorf("App Instance %s-%s: Error(%d): %s",
 				config.DisplayName, config.UUIDandVersion.UUID, i, errStr)
 		}

pkg/pillar/cmd/zedrouter/appnetwork.go

@@ -102,6 +102,9 @@ func (z *zedrouter) prepareConfigForVIFs(config types.AppNetworkConfig,
 		}
 		adapterStatus.HostName = config.Key()
 		adapterStatus.MTU = netInstStatus.MTU
+		// Propagate IntfOrder from adapter down to VifConfig, which zedmanager then passes
+		// to domainmgr.
+		adapterStatus.VifConfig.VifOrder = adapterStatus.IntfOrder
 		guestIP, err := z.lookupOrAllocateIPv4ForVIF(
 			netInstStatus, *adapterStatus, status.UUIDandVersion.UUID)
 		if err != nil {

pkg/pillar/types/domainmgrtypes.go

@@ -261,6 +261,8 @@ type VmConfig struct {
 	CPUsPinned         bool
 	VMMMaxMem          int // in kbytes
 	EnableVncShimVM    bool
+	// Enables enforcement of user-defined ordering for network interfaces.
+	EnforceNetworkInterfaceOrder bool
 }
 
 // VmMode is the type for the virtualization mode
@@ -425,6 +427,10 @@ type VifConfig struct {
 	MTU    uint16
 	// PodVif is only valid in the Kubernetes mode.
 	PodVif PodVIF
+	// Interface order across both VIFs and directly attached network devices.
+	// Note that we cannot use attribute name "IntfOrder" here, otherwise it would
+	// overlap with IntfOrder from AppNetAdapterConfig inside AppNetAdapterStatus.
+	VifOrder uint32
 }
 
 // PodVIF : configuration parameters for VIF connecting Kubernetes pod with the host.

pkg/pillar/types/zedmanagertypes.go

@@ -152,9 +152,10 @@ type AppInstanceOpsCmd struct {
 
 // IoAdapter specifies that a group of ports should be assigned
 type IoAdapter struct {
-	Type  IoType
-	Name  string      // Short hand name such as "COM1" or "eth1-2"
-	EthVf sriov.EthVF // Applies only to the VF IoType
+	Type      IoType
+	Name      string      // Short hand name such as "COM1" or "eth1-2"
+	EthVf     sriov.EthVF // Applies only to the VF IoType
+	IntfOrder uint32      // Interface order across both virtual and passthrough network devices.
 }
 
 // LogCreate :

pkg/pillar/types/zedroutertypes.go

@@ -282,7 +282,7 @@ type AppNetAdapterConfig struct {
 	Name       string           // From proto message
 	AppMacAddr net.HardwareAddr // If set use it for vif
 	AppIPAddr  net.IP           // If set use DHCP to assign to app
-	IntfOrder  int32            // XXX need to get from API
+	IntfOrder  uint32           // Order wrt. other virtual and also directly assigned network adapters
 
 	// XXX Shouldn't we use ErrorAndTime here
 	// Error