More ntp servers

Makefile

@@ -249,6 +249,12 @@ QEMU_TPM_DEVICE_riscv64=tpm-tis
 QEMU_OPTS_TPM_Y_$(ZARCH)=-chardev socket,id=chrtpm,path=$(CURRENT_SWTPM)/swtpm-sock -tpmdev emulator,id=tpm0,chardev=chrtpm -device $(QEMU_TPM_DEVICE_$(ZARCH)),tpmdev=tpm0
 QEMU_OPTS_TPM=$(QEMU_OPTS_TPM_$(TPM:%=Y)_$(ZARCH))
 
+ifneq ($(TAP),)
+QEMU_OPTS_eth1=-netdev tap,id=eth1,ifname=$(TAP),script="" -device virtio-net-pci,netdev=eth1,romfile=""
+else
+QEMU_OPTS_eth1=-netdev user,id=eth1,net=$(QEMU_OPTS_NET2),dhcpstart=$(QEMU_OPTS_NET2_FIRST_IP) -device virtio-net-pci,netdev=eth1,romfile=""
+endif
+
 QEMU_OPTS_amd64=-smbios type=1,serial=$(QEMU_EVE_SERIAL)
 QEMU_OPTS_arm64=-smbios type=1,serial=$(QEMU_EVE_SERIAL) -drive file=fat:rw:$(dir $(DEVICETREE_DTB)),label=QEMU_DTB,format=vvfat
 QEMU_OPTS_riscv64=-kernel $(UBOOT_IMG)/u-boot.bin -device virtio-blk,drive=uefi-disk
@@ -259,7 +265,7 @@ QEMU_OPTS_COMMON= -m $(QEMU_MEMORY) -smp 4  $(QEMU_OPTS_BIOS) \
         -global ICH9-LPC.noreboot=false -watchdog-action reset \
         -rtc base=utc,clock=rt \
         -netdev user,id=eth0,net=$(QEMU_OPTS_NET1),dhcpstart=$(QEMU_OPTS_NET1_FIRST_IP),hostfwd=tcp::$(SSH_PORT)-:22$(QEMU_TFTP_OPTS) -device virtio-net-pci,netdev=eth0,romfile="" \
-        -netdev user,id=eth1,net=$(QEMU_OPTS_NET2),dhcpstart=$(QEMU_OPTS_NET2_FIRST_IP) -device virtio-net-pci,netdev=eth1,romfile="" \
+	$(QEMU_OPTS_eth1) \
         -device nec-usb-xhci,id=xhci \
         -qmp unix:$(CURDIR)/qmp.sock,server,wait=off
 QEMU_OPTS_CONF_PART=$(shell [ -d "$(CONF_PART)" ] && echo '-drive file=fat:rw:$(CONF_PART),format=raw')

pkg/pillar/cmd/diag/diag.go

@@ -961,8 +961,8 @@ func printOutput(ctx *diagContext, caller string) {
 			}
 			ctx.ph.Print("INFO: %s: Static Domain Name: %s\n",
 				ifname, port.DomainName)
-			ctx.ph.Print("INFO: %s: Static NTP server: %s\n",
-				ifname, port.NtpServer.String())
+			ctx.ph.Print("INFO: %s: Static NTP server: %+v\n",
+				ifname, port.ConfiguredNtpServers)
 		}
 		printProxy(ctx, port, ifname)
 		if port.HasError() {

pkg/pillar/cmd/zedagent/handlemetrics.go

@@ -1106,10 +1106,7 @@ func PublishAppInfoToZedCloud(ctx *zedagentContext, uuid string,
 			networkInfo.DevName = *proto.String(name)
 			niStatus := appIfnameToNetworkInstance(ctx, aiStatus, ifname)
 			if niStatus != nil {
-				for _, ntpServer := range niStatus.NTPServers {
-					networkInfo.NtpServers = append(networkInfo.NtpServers,
-						ntpServer.String())
-				}
+				networkInfo.NtpServers = append(networkInfo.NtpServers, niStatus.NTPServers...)
 				networkInfo.DefaultRouters = []string{niStatus.Gateway.String()}
 				networkInfo.Dns = &info.ZInfoDNS{
 					DNSservers: []string{},

pkg/pillar/cmd/zedagent/parseconfig.go

@@ -1380,10 +1380,11 @@ func parseOneSystemAdapterConfig(getconfigCtx *getconfigContext,
 			port.WirelessCfg = network.WirelessCfg
 			port.Gateway = network.Gateway
 			port.DomainName = network.DomainName
-			port.NTPServer = network.NTPServer
+			port.NTPServers = network.NTPServers
 			port.DNSServers = network.DNSServers
 			// Need to be careful since zedcloud can feed us bad Dhcp type
 			port.Dhcp = network.Dhcp
+			port.IgnoreDhcpNtpServers = network.IgnoreDhcpNtpServers
 			switch port.Dhcp {
 			case types.DhcpTypeStatic:
 				if sysAdapter.Addr == "" {
@@ -2260,12 +2261,18 @@ func parseIpspecNetworkXObject(ipspec *zconfig.Ipspec, config *types.NetworkXObj
 			return fmt.Errorf("invalid gateway IP (%s)", g)
 		}
 	}
-	if n := ipspec.GetNtp(); n != "" {
-		config.NTPServer = net.ParseIP(n)
-		if config.NTPServer == nil {
-			return fmt.Errorf("invalid NTP IP (%s)", n)
-		}
+
+	ntpServers := append([]string{ipspec.GetNtp()}, ipspec.GetMoreNtp()...)
+	if len(ntpServers) > 0 && ntpServers[0] != "" {
+		config.NTPServers = ntpServers
+	}
+
+	config.IgnoreDhcpNtpServers = false
+	dhcpOptionsIgnore := ipspec.GetDhcpOptionsIgnore()
+	if dhcpOptionsIgnore != nil {
+		config.IgnoreDhcpNtpServers = dhcpOptionsIgnore.NtpServerExclusively
 	}
+
 	for _, dsStr := range ipspec.GetDns() {
 		ds := net.ParseIP(dsStr)
 		if ds == nil {
@@ -2293,11 +2300,11 @@ func parseIpspec(ipspec *zconfig.Ipspec,
 
 	config.DomainName = ipspec.GetDomain()
 	// Parse NTP Server
+	if config.NtpServers == nil {
+		config.NtpServers = make([]string, 0)
+	}
 	if n := ipspec.GetNtp(); n != "" {
-		config.NtpServer = net.ParseIP(n)
-		if config.NtpServer == nil {
-			return fmt.Errorf("invalid NTP IP (%s)", n)
-		}
+		config.NtpServers = append(config.NtpServers, n)
 	}
 	// Parse Dns Servers
 	for _, dsStr := range ipspec.GetDns() {

pkg/pillar/cmd/zedagent/parseconfig_test.go

@@ -147,7 +147,7 @@ func TestParsePhysicalNetworkAdapters(t *testing.T) {
 	g.Expect(port.DhcpConfig.AddrSubnet).To(BeEmpty())
 	g.Expect(port.DhcpConfig.DNSServers).To(BeEmpty())
 	g.Expect(port.DhcpConfig.Gateway).To(BeNil())
-	g.Expect(port.DhcpConfig.NTPServer).To(BeNil())
+	g.Expect(port.DhcpConfig.NTPServers).To(BeNil())
 	g.Expect(port.ProxyConfig.Proxies).To(BeEmpty())
 	g.Expect(port.L2LinkConfig.L2Type).To(Equal(types.L2LinkTypeNone))
 	g.Expect(port.WirelessCfg.WType).To(Equal(types.WirelessTypeNone))

pkg/pillar/cmd/zedagent/reportinfo.go

@@ -641,7 +641,7 @@ func PublishDeviceInfoToZedCloud(ctx *zedagentContext, dest destinationBitset) {
 	// device returns a runtime error. Similarly, we only support enforced application network
 	// interface order for the KVM hypervisor. If enabled for application deployed under Xen
 	// or Kubevirt hypervisor, EVE returns error and the application will not be started.
-	ReportDeviceInfo.ApiCapability = info.APICapability_API_CAPABILITY_ENFORCED_NET_INTERFACE_ORDER
+	ReportDeviceInfo.ApiCapability = info.APICapability_API_CAPABILITY_NTPS_FQDN
 
 	// Report if there is a local override of profile
 	if ctx.getconfigCtx.sideController.currentProfile !=
@@ -885,8 +885,10 @@ func encodeNetInfo(port types.NetworkPortStatus) *info.ZInfoNetwork {
 
 	networkInfo.Proxy = encodeProxyStatus(&port.ProxyConfig)
 	networkInfo.NtpServers = []string{}
-	for _, server := range port.NtpServers {
-		networkInfo.NtpServers = append(networkInfo.NtpServers, server.String())
+	if !port.IgnoreDhcpNtpServers {
+		for _, server := range port.DhcpNtpServers {
+			networkInfo.NtpServers = append(networkInfo.NtpServers, server.String())
+		}
 	}
 	return networkInfo
 }
@@ -1035,12 +1037,25 @@ func encodeNetworkPortStatus(ctx *zedagentContext,
 	devicePort.Up = port.Up
 	devicePort.Mtu = uint32(port.MTU)
 	devicePort.Domainname = port.DomainName
-	// TODO: modify EVE APIs and allow to publish full list of NTP servers
-	if port.NtpServer != nil {
-		devicePort.NtpServer = port.NtpServer.String()
-	} else if len(port.NtpServers) > 0 {
-		devicePort.NtpServer = port.NtpServers[0].String()
+	ntpServers := make([]string, 0)
+	if port.ConfiguredNtpServers != nil {
+		ntpServers = append(ntpServers, port.ConfiguredNtpServers...)
+	}
+	if len(port.DhcpNtpServers) > 0 && !port.IgnoreDhcpNtpServers {
+		ntpServers = append(ntpServers, port.DhcpNtpServers[0].String())
+		if len(port.DhcpNtpServers) > 1 {
+			for _, ntpServer := range port.DhcpNtpServers[1:] {
+				ntpServers = append(ntpServers, ntpServer.String())
+			}
+		}
+	}
+	if len(ntpServers) > 0 {
+		devicePort.NtpServer = ntpServers[0]
+	}
+	if len(ntpServers) > 1 {
+		devicePort.MoreNtpServers = ntpServers[1:]
 	}
+
 	devicePort.Proxy = encodeProxyStatus(&port.ProxyConfig)
 	devicePort.MacAddr = port.MacAddr.String()
 	for _, ipAddr := range port.AddrInfoList {
@@ -1143,7 +1158,13 @@ func encodeNetworkPortConfig(ctx *zedagentContext,
 	dp.DefaultRouters = make([]string, 0)
 	dp.DefaultRouters = append(dp.DefaultRouters, npc.Gateway.String())
 
-	dp.NtpServer = npc.NTPServer.String()
+	if npc.NTPServers != nil && len(npc.NTPServers) > 0 {
+		dp.NtpServer = npc.NTPServers[0]
+
+		if len(npc.NTPServers) > 1 {
+			dp.MoreNtpServers = append(dp.MoreNtpServers, npc.NTPServers[1:]...)
+		}
+	}
 
 	dp.Dns = new(info.ZInfoDNS)
 	dp.Dns.DNSdomain = npc.DomainName

pkg/pillar/cmd/zedrouter/networkinstance.go

@@ -10,6 +10,7 @@ import (
 	"net"
 	"strings"
 
+	"github.com/lf-edge/eve/pkg/pillar/devicenetwork"
 	"github.com/lf-edge/eve/pkg/pillar/nireconciler"
 	"github.com/lf-edge/eve/pkg/pillar/nistate"
 	"github.com/lf-edge/eve/pkg/pillar/types"
@@ -92,6 +93,41 @@ func (z *zedrouter) getNIBridgeConfig(
 	}
 }
 
+func (z *zedrouter) attachNTPServersToPortConfigs(portConfigs []nireconciler.Port) {
+	for i := range portConfigs {
+		pc := &portConfigs[i]
+		ntpServerIPs, ntpServerDomainsOrIPs := types.GetNTPServers(*z.deviceNetworkStatus, pc.IfName)
+
+		ntpServers := make([]net.IP, 0, len(ntpServerDomainsOrIPs))
+		for _, ntpServer := range ntpServerDomainsOrIPs {
+			ip := net.ParseIP(ntpServer)
+			if ip != nil {
+				ntpServers = append(ntpServers, ip)
+				continue
+			}
+			z.pubSub.StillRunning(agentName, warningTime, errorTime)
+			dnsResponses, err := devicenetwork.ResolveWithPortsLambda(
+				ntpServer,
+				*z.deviceNetworkStatus,
+				devicenetwork.ResolveCacheWrap(devicenetwork.ResolveWithSrcIP),
+			)
+			if err != nil {
+				z.log.Warnf("could not resolve '%s': %v", ntpServer, err)
+			}
+
+			for _, dnsResponse := range dnsResponses {
+				ntpServers = append(ntpServers, dnsResponse.IP)
+			}
+		}
+
+		ntpServers = append(ntpServers, ntpServerIPs...)
+		generics.FilterDuplicatesFn(ntpServers, netutils.EqualIPs)
+
+		pc.NTPServers = ntpServers
+	}
+}
+
+// NTP servers are set separately with z.attachNTPServersToPortConfigs
 func (z *zedrouter) getNIPortConfig(
 	status *types.NetworkInstanceStatus) (portConfigs []nireconciler.Port) {
 	if len(status.Ports) == 0 {
@@ -111,7 +147,6 @@ func (z *zedrouter) getNIPortConfig(
 			MTU:          port.MTU,
 			DhcpType:     port.Dhcp,
 			DNSServers:   types.GetDNSServers(*z.deviceNetworkStatus, port.IfName),
-			NTPServers:   types.GetNTPServers(*z.deviceNetworkStatus, port.IfName),
 		})
 	}
 	return portConfigs
@@ -124,12 +159,12 @@ func (z *zedrouter) updateNIPorts(niConfig types.NetworkInstanceConfig,
 	var (
 		newPorts         []*types.NetworkPortStatus
 		validatedPortLLs []string
-		newNTPServers    []net.IP
+		newNTPServers    []string
 		errorMsgs        []string
 	)
-	if niStatus.NtpServer != nil {
+	if niStatus.NtpServers != nil {
 		// The NTP server explicitly configured for the NI.
-		newNTPServers = append(newNTPServers, niStatus.NtpServer)
+		newNTPServers = append(newNTPServers, niStatus.NtpServers...)
 	}
 	if niStatus.PortLabel != "" {
 		newPorts = z.deviceNetworkStatus.LookupPortsByLabel(niStatus.PortLabel)
@@ -205,23 +240,26 @@ func (z *zedrouter) updateNIPorts(niConfig types.NetworkInstanceConfig,
 		}
 		// Port is valid for this network instance.
 		validatedPortLLs = append(validatedPortLLs, port.Logicallabel)
-		if port.NtpServer != nil {
+		if port.ConfiguredNtpServers != nil {
 			// The NTP server explicitly configured for the port.
-			newNTPServers = append(newNTPServers, port.NtpServer)
+			newNTPServers = append(newNTPServers, port.ConfiguredNtpServers...)
 		}
 		// NTP servers received via DHCP.
-		newNTPServers = append(newNTPServers, port.NtpServers...)
+		if !port.IgnoreDhcpNtpServers {
+			for _, dhcpNtpserver := range port.DhcpNtpServers {
+				newNTPServers = append(newNTPServers, dhcpNtpserver.String())
+			}
+		}
 	}
 	if niStatus.PortLabel != "" && len(newPorts) == 0 {
 		// This is potentially a transient state, wait for DNS update.
 		errorMsgs = append(errorMsgs,
 			fmt.Sprintf("no port is matching label '%s'", niStatus.PortLabel))
 	}
-	newNTPServers = generics.FilterDuplicatesFn(newNTPServers, netutils.EqualIPs)
+	newNTPServers = generics.FilterDuplicates(newNTPServers)
 	changed = changed || !generics.EqualSets(niStatus.Ports, validatedPortLLs)
 	niStatus.Ports = validatedPortLLs
-	changed = changed || !generics.EqualSetsFn(niStatus.NTPServers, newNTPServers,
-		netutils.EqualIPs)
+	changed = changed || !generics.EqualSets(niStatus.NTPServers, newNTPServers)
 	niStatus.NTPServers = newNTPServers
 	// Update BridgeMac for Switch NI bridge created by NIM.
 	if z.niBridgeIsCreatedByNIM(niConfig) {
@@ -492,8 +530,10 @@ func (z *zedrouter) updateNIRoutePort(route types.IPRouteConfig, port string,
 func (z *zedrouter) doActivateNetworkInstance(config types.NetworkInstanceConfig,
 	status *types.NetworkInstanceStatus) {
 	// Create network instance inside the network stack.
+	bridgeConfig := z.getNIBridgeConfig(status)
+	z.attachNTPServersToPortConfigs(bridgeConfig.Ports)
 	niRecStatus, err := z.niReconciler.AddNI(
-		z.runCtx, config, z.getNIBridgeConfig(status))
+		z.runCtx, config, bridgeConfig)
 	if err != nil {
 		z.log.Errorf("Failed to activate network instance %s: %v", status.UUID, err)
 		status.ReconcileErr.SetErrorNow(err.Error())
@@ -537,8 +577,10 @@ func (z *zedrouter) doInactivateNetworkInstance(status *types.NetworkInstanceSta
 
 func (z *zedrouter) doUpdateActivatedNetworkInstance(config types.NetworkInstanceConfig,
 	status *types.NetworkInstanceStatus) {
+	bridgeConfig := z.getNIBridgeConfig(status)
+	z.attachNTPServersToPortConfigs(bridgeConfig.Ports)
 	niRecStatus, err := z.niReconciler.UpdateNI(
-		z.runCtx, config, z.getNIBridgeConfig(status))
+		z.runCtx, config, bridgeConfig)
 	if err != nil {
 		z.log.Errorf("Failed to update activated network instance %s: %v",
 			status.UUID, err)

pkg/pillar/cmd/zedrouter/pubsubhandlers.go

@@ -547,6 +547,19 @@ func (z *zedrouter) handleAppNetworkModify(ctxArg interface{}, key string,
 	configArg interface{}, oldConfigArg interface{}) {
 	newConfig := configArg.(types.AppNetworkConfig)
 	oldConfig := oldConfigArg.(types.AppNetworkConfig)
+
+	// re-activate network instances of edge apps in order to resolve NTP servers again
+	for _, appNetConfig := range newConfig.AppNetAdapterList {
+		appNetStatus := z.lookupNetworkInstanceStatus(appNetConfig.Network.String())
+		if appNetStatus == nil {
+			continue
+		}
+		if appNetStatus.Activated {
+			appNetConfig := z.lookupNetworkInstanceConfig(appNetStatus.Key())
+			z.doUpdateActivatedNetworkInstance(*appNetConfig, appNetStatus)
+		}
+	}
+
 	status := z.lookupAppNetworkStatus(key)
 	z.log.Functionf("handleAppNetworkModify(%v) for %s",
 		newConfig.UUIDandVersion, newConfig.DisplayName)

pkg/pillar/devicenetwork/dns.go

@@ -5,6 +5,7 @@ package devicenetwork
 
 import (
 	"fmt"
+	"math"
 	"net"
 	"os"
 	"path/filepath"
@@ -96,6 +97,50 @@ func ResolveWithSrcIP(domain string, dnsServerIP net.IP, srcIP net.IP) ([]DNSRes
 	return response, nil
 }
 
+type cachedDNSResponses struct {
+	dnsResponses []DNSResponse
+	validUntil   time.Time
+}
+
+type cachedDNSResponseKey struct {
+	domain string
+	srcIP  string
+}
+
+var resolveCache = map[cachedDNSResponseKey]cachedDNSResponses{}
+
+// ResolveCacheWrap wraps around a resolve func (e.g. ResolveWithSrcIP) and caches DNS entries
+func ResolveCacheWrap(resolve func(string, net.IP, net.IP) ([]DNSResponse, error)) func(domain string, dnsServerIP net.IP, srcIP net.IP) ([]DNSResponse, error) {
+	return func(domain string, dnsServerIP net.IP, srcIP net.IP) ([]DNSResponse, error) {
+
+		cacheKey := cachedDNSResponseKey{
+			domain: domain,
+			srcIP:  srcIP.String(),
+		}
+		dnsResponses, found := resolveCache[cacheKey]
+		if !found || !dnsResponses.validUntil.After(time.Now()) {
+			dnsResponses, err := resolve(domain, dnsServerIP, srcIP)
+			if err == nil {
+				minValidUntil := uint32(math.MaxUint32)
+				for _, dnsResponse := range dnsResponses {
+					if dnsResponse.TTL < uint32(minValidUntil) {
+						minValidUntil = dnsResponse.TTL
+					}
+				}
+				validUntil := time.Now().Add(time.Duration(minValidUntil * uint32(time.Second)))
+				resolveCache[cacheKey] = cachedDNSResponses{
+					dnsResponses: dnsResponses,
+					validUntil:   validUntil,
+				}
+			}
+
+			return dnsResponses, err
+		}
+
+		return dnsResponses.dnsResponses, nil
+	}
+}
+
 // ResolveWithPortsLambda resolves a domain by using source IPs and dns servers from DeviceNetworkStatus
 // As a resolver func ResolveWithSrcIP can be used
 func ResolveWithPortsLambda(domain string,