[network] interface-aware DNS lookups

hal/inc/hal_dynalib_netdb_posix.h

@@ -38,6 +38,7 @@ DYNALIB_FN(1, hal_netdb, netdb_gethostbyname_r, int(const char*, struct hostent*
 DYNALIB_FN(2, hal_netdb, netdb_freeaddrinfo, void(struct addrinfo*))
 DYNALIB_FN(3, hal_netdb, netdb_getaddrinfo, int(const char*, const char*, const struct addrinfo*, struct addrinfo**))
 DYNALIB_FN(4, hal_netdb, netdb_getnameinfo, int(const struct sockaddr*, socklen_t, char*, socklen_t, char*, socklen_t, int))
+DYNALIB_FN(5, hal_netdb, netdb_getaddrinfo_ex, int(const char*, const char*, const struct addrinfo*, struct addrinfo**, if_t))
 
 DYNALIB_END(hal_netdb)
 

hal/inc/netdb_hal.h

@@ -26,6 +26,8 @@
 
 #include "netdb_hal_impl.h"
 
+#include "ifapi.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif /* __cplusplus */
@@ -91,6 +93,9 @@ void netdb_freeaddrinfo(struct addrinfo* ai);
 int netdb_getaddrinfo(const char* hostname, const char* servname,
                       const struct addrinfo* hints, struct addrinfo** res);
 
+int netdb_getaddrinfo_ex(const char* hostname, const char* servname,
+                      const struct addrinfo* hints, struct addrinfo** res, if_t iface);
+
 /**
  * Converts a sockaddr structure to a pair of host name and service strings.
  *

hal/network/api/ifapi.h

@@ -260,6 +260,17 @@ typedef enum if_req_t {
     IF_REQ_DHCP_SETTINGS = 4,
 } if_req_t;
 
+typedef enum if_protocol_state_t {
+    IF_PROTOCOL_STATE_UNCONFIGURED = 0,
+    IF_PROTOCOL_STATE_LINK_LOCAL = 1,
+    IF_PROTOCOL_STATE_CONFIGURED = 2
+} if_protocol_state_t;
+
+typedef struct if_protocol_state {
+    if_protocol_state_t ipv4;
+    if_protocol_state_t ipv6;
+} if_protocol_state;
+
 int if_init(void);
 int if_init_platform(void*);
 int if_init_platform_postpone(void*);
@@ -315,6 +326,8 @@ int if_get_power_state(if_t iface, if_power_state_t* state);
 
 int if_get_profile(if_t iface, char* profile, size_t length);
 
+int if_get_protocol_state(if_t iface, if_protocol_state* state, void* reserved);
+
 #ifdef __cplusplus
 }
 #endif /* __cplusplus */

hal/network/lwip/ifapi.cpp

@@ -37,6 +37,7 @@ extern "C" {
 #include "check.h"
 
 #include "wiznet/wiznetif_config.h"
+#include "scope_guard.h"
 
 using namespace particle::net;
 
@@ -1310,3 +1311,74 @@ int if_get_profile(if_t iface, char* profile, size_t length) {
     }
     return p.size();
 }
+
+// FIXME: duplicate of refreshIpState() in NetworkManager
+int if_get_protocol_state(if_t iface, if_protocol_state* state, void* reserved) {
+    CHECK_TRUE(state, SYSTEM_ERROR_INVALID_ARGUMENT);
+    if_addrs* addrs = nullptr;
+    CHECK(if_get_if_addrs(&addrs));
+    SCOPE_GUARD({
+        if_free_if_addrs(addrs);
+    });
+
+    uint8_t idx = 0;
+    if (iface) {
+        CHECK(if_get_index(iface, &idx));
+    }
+
+    state->ipv4 = IF_PROTOCOL_STATE_UNCONFIGURED;
+    state->ipv6 = IF_PROTOCOL_STATE_UNCONFIGURED;
+
+    for (auto addr = addrs; addr != nullptr; addr = addr->next) {
+        if (iface && addr->ifindex != idx) {
+            continue;
+        }
+
+        /* Skip loopback interface */
+        if (addr->ifflags & IFF_LOOPBACK) {
+            continue;
+        }
+
+        if (addr->ifflags & IFF_DEBUG) {
+            continue;
+        }
+
+        /* Skip non-UP and non-LINK_UP interfaces */
+        if ((addr->ifflags & (IFF_UP | IFF_LOWER_UP)) != (IFF_UP | IFF_LOWER_UP)) {
+            continue;
+        }
+
+        auto a = addr->if_addr;
+        if (!a || !a->addr) {
+            continue;
+        }
+
+        if (a->addr->sa_family == AF_INET) {
+            if (a->prefixlen > 0 && /* FIXME */ a->gw && ((sockaddr_in*)a->gw)->sin_addr.s_addr != INADDR_ANY) {
+                state->ipv4 = IF_PROTOCOL_STATE_CONFIGURED;
+            }
+        } else if (a->addr->sa_family == AF_INET6) {
+            sockaddr_in6* sin6 = (sockaddr_in6*)a->addr;
+            auto ip6_addr_data = a->ip6_addr_data;
+
+            /* NOTE: we say that IPv6 is configured if there is at least
+             * one IPv6 address on an interface without scope and in a VALID state,
+             * which is in fact either PREFERRED or DEPRECATED.
+             */
+            if (sin6->sin6_scope_id == 0 && a->prefixlen > 0 &&
+                    ip6_addr_data && (ip6_addr_data->state & IF_IP6_ADDR_STATE_VALID)) {
+                state->ipv6 = IF_PROTOCOL_STATE_CONFIGURED;
+            } else if (sin6->sin6_scope_id != 0 && a->prefixlen > 0 &&
+                    ip6_addr_data && (ip6_addr_data->state & IF_IP6_ADDR_STATE_VALID)) {
+                // Otherwise report link-local
+                if (state->ipv6 != IF_PROTOCOL_STATE_CONFIGURED) {
+                    state->ipv6 = IF_PROTOCOL_STATE_LINK_LOCAL;
+                }
+            }
+        } else {
+            /* Unknown family */
+        }
+    }
+
+    return 0;
+}

hal/network/lwip/netdb_hal.cpp

@@ -26,6 +26,7 @@
 #include <lwip/sockets.h>
 #include <errno.h>
 #include <algorithm>
+#include "resolvapi.h"
 
 struct hostent* netdb_gethostbyname(const char *name) {
     return lwip_gethostbyname(name);
@@ -42,26 +43,39 @@ void netdb_freeaddrinfo(struct addrinfo* ai) {
 
 int netdb_getaddrinfo(const char* hostname, const char* servname,
                       const struct addrinfo* hints, struct addrinfo** res) {
-    /* Change the behavior when AF_UNSPEC is used */
-    if (hints && hints->ai_family == AF_UNSPEC) {
-        struct addrinfo h = *hints;
-
-        /* First perform a lookup with AF_INET6 */
-        h.ai_family = AF_INET6;
-        int rinet6 = lwip_getaddrinfo(hostname, servname, &h, res);
+    return netdb_getaddrinfo_ex(hostname, servname, hints, res, nullptr);
+}
 
-        /* Next perform a lookup with AF_INET */
-        h.ai_family = AF_INET;
-        /* FIXME: expects that there is either 1 or 0 results from the previous call */
-        int rinet = lwip_getaddrinfo(hostname, servname, &h, rinet6 == 0 && *res ? &((*res)->ai_next) : res);
+int netdb_getaddrinfo_ex(const char* hostname, const char* servname,
+                         const struct addrinfo* hints, struct addrinfo** res, if_t iface) {
+    uint8_t ifaceIndex = 0;
+    if (iface) {
+        if_get_index(iface, &ifaceIndex);
+    }
+    if (hints && hints->ai_family == AF_UNSPEC /* && hints->ai_flags & AI_ADDRCONFIG ?*/) {
+        // For now defaulting as if AI_ADDRCONFIG is always set in case of AF_UNSPEC, simplifies things
+        if_protocol_state state = {};
+        if_get_protocol_state(iface, &state, nullptr);
+        if (state.ipv6 != IF_PROTOCOL_STATE_UNCONFIGURED) {
+            struct addrinfo h = *hints;
+
+            /* First perform a lookup with AF_INET6 */
+            h.ai_family = AF_INET6;
+            int rinet6 = lwip_getaddrinfo_ex(hostname, servname, &h, res, ifaceIndex);
+
+            /* Next perform a lookup with AF_INET */
+            h.ai_family = AF_INET;
+            /* FIXME: expects that there is either 1 or 0 results from the previous call */
+            int rinet = lwip_getaddrinfo_ex(hostname, servname, &h, rinet6 == 0 && *res ? &((*res)->ai_next) : res, ifaceIndex);
+
+            if (rinet6 == 0 || rinet == 0) {
+                return 0;
+            }
 
-        if (rinet6 == 0 || rinet == 0) {
-            return 0;
+            return std::max(rinet, rinet6);
         }
-
-        return std::max(rinet, rinet6);
     }
-    return lwip_getaddrinfo(hostname, servname, hints, res);
+    return lwip_getaddrinfo_ex(hostname, servname, hints, res, ifaceIndex);
 }
 
 int netdb_getnameinfo(const struct sockaddr* sa, socklen_t salen, char* host,

system/src/system_cloud_connection.h

@@ -27,6 +27,7 @@
 #if HAL_PLATFORM_IFAPI
 #include "netdb_hal.h"
 #endif // HAL_PLATFORM_IFAPI
+#include "system_network.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -55,7 +56,7 @@ int system_cloud_get_inet_family_keepalive(int af, unsigned int* value);
 sock_handle_t system_cloud_get_socket_handle();
 
 #if HAL_PLATFORM_IFAPI
-int system_cloud_resolv_address(int protocol, const ServerAddress* address, sockaddr* saddrCache, addrinfo** info, CloudServerAddressType* type, bool useCachedAddrInfo);
+int system_cloud_resolv_address(int protocol, const ServerAddress* address, sockaddr* saddrCache, addrinfo** info, CloudServerAddressType* type, bool useCachedAddrInfo, network_handle_t interface = NETWORK_INTERFACE_ALL, bool flushDnsCache = false);
 #endif // HAL_PLATFORM_IFAPI
 
 #ifdef __cplusplus

system/src/system_cloud_connection_posix.cpp

@@ -47,7 +47,7 @@ const unsigned CLOUD_SOCKET_HALF_CLOSED_WAIT_TIMEOUT = 5000;
 
 } /* anonymous */
 
-int system_cloud_resolv_address(int protocol, const ServerAddress* address, sockaddr* saddrCache, addrinfo** info, CloudServerAddressType* type, bool useCachedAddrInfo) {
+int system_cloud_resolv_address(int protocol, const ServerAddress* address, sockaddr* saddrCache, addrinfo** info, CloudServerAddressType* type, bool useCachedAddrInfo, network_handle_t interface, bool flushDnsCache) {
     CHECK_TRUE(info, SYSTEM_ERROR_INVALID_ARGUMENT);
 
     *type = CLOUD_SERVER_ADDRESS_TYPE_NONE;
@@ -117,8 +117,15 @@ int system_cloud_resolv_address(int protocol, const ServerAddress* address, sock
                 /* FIXME: this should probably be moved into system_cloud_internal */
                 system_string_interpolate(address->domain, tmphost, sizeof(tmphost), system_interpolate_cloud_server_hostname);
                 snprintf(tmpserv, sizeof(tmpserv), "%u", address->port);
-                LOG(TRACE, "Resolving %s#%s", tmphost, tmpserv);
-                netdb_getaddrinfo(tmphost, tmpserv, &hints, info);
+                if (flushDnsCache) {
+                    hints.ai_flags |= AI_FLUSHCACHE;
+                }
+                if_t iface = nullptr;
+                if (interface != NETWORK_INTERFACE_ALL) {
+                    if_get_by_index(interface, &iface);
+                }
+                LOG(TRACE, "Resolving %s#%s cache=%d iface=%d", tmphost, tmpserv, !flushDnsCache, interface);
+                netdb_getaddrinfo_ex(tmphost, tmpserv, &hints, info, iface);
                 *type = CLOUD_SERVER_ADDRESS_TYPE_NEW_ADDRINFO;
                 break;
             }
@@ -139,7 +146,9 @@ int system_cloud_connect(int protocol, const ServerAddress* address, sockaddr* s
     CloudServerAddressType type = CLOUD_SERVER_ADDRESS_TYPE_NONE;
     bool clean = true;
 
-    system_cloud_resolv_address(protocol, address, saddrCache, &info, &type, true /* useCachedAddrInfo */);
+    network_interface_t cloudInterface = particle::system::ConnectionManager::instance()->selectCloudConnectionNetwork();
+
+    system_cloud_resolv_address(protocol, address, saddrCache, &info, &type, true /* useCachedAddrInfo */, cloudInterface);
 
     int r = SYSTEM_ERROR_NETWORK;
 
@@ -148,6 +157,21 @@ int system_cloud_connect(int protocol, const ServerAddress* address, sockaddr* s
     for (struct addrinfo* a = info; a != nullptr; a = a->ai_next) {
         /* Iterate over all the addresses and attempt to connect */
 
+        switch (a->ai_family) {
+            case AF_INET: {
+                if (!network_ready(cloudInterface, NETWORK_READY_TYPE_IPV4, nullptr)) {
+                    continue;
+                }
+                break;
+            }
+            case AF_INET6: {
+                if (!network_ready(cloudInterface, NETWORK_READY_TYPE_IPV6, nullptr)) {
+                    continue;
+                }
+                break;
+            }
+        }
+
         int s = sock_socket(a->ai_family, a->ai_socktype, a->ai_protocol);
         if (s < 0) {
             LOG(ERROR, "Cloud socket failed, family=%d, type=%d, protocol=%d, errno=%d", a->ai_family, a->ai_socktype, a->ai_protocol, errno);
@@ -204,8 +228,6 @@ int system_cloud_connect(int protocol, const ServerAddress* address, sockaddr* s
             }
         }
 
-        network_interface_t cloudInterface = particle::system::ConnectionManager::instance()->selectCloudConnectionNetwork();
-
         LOG(INFO, "Cloud socket=%d, connecting to %s#%u using if %d", s, serverHost, serverPort, cloudInterface);
 
         if (cloudInterface != NETWORK_INTERFACE_ALL) {    

system/src/system_connection_manager.cpp

@@ -70,16 +70,18 @@ bool isValidScore(uint32_t score) {
     return std::numeric_limits<uint32_t>::max() != score;
 }
 
-static int getCloudHostnameAndPort(addrinfo** info, CloudServerAddressType* type, bool allowCached = false) {
+static int getCloudHostnameAndPort(addrinfo** info, CloudServerAddressType* type, bool allowCached = false, network_interface_t interface = NETWORK_INTERFACE_ALL) {
     ServerAddress server_addr = {};
     HAL_FLASH_Read_ServerAddress(&server_addr);
     if (server_addr.port == 0 || server_addr.port == 0xFFFF) {
         server_addr.port = spark_cloud_udp_port_get();
     }
 
-    return system_cloud_resolv_address(IPPROTO_UDP, &server_addr, allowCached ? (sockaddr*)&g_system_cloud_session_data.address : nullptr, info, type, false /* useCachedAddrInfo */);
+    return system_cloud_resolv_address(IPPROTO_UDP, &server_addr, allowCached ? (sockaddr*)&g_system_cloud_session_data.address : nullptr, info, type, false /* useCachedAddrInfo */, interface, !allowCached /* flushDnsCache*/);
 }
 
+size_t ConnectionTester::roundRobinInterfaceForDns_ = 0;
+
 ConnectionManager::ConnectionManager()
     : preferredNetwork_(NETWORK_INTERFACE_ALL) {
     bestNetworks_ = ConnectionTester::getSupportedInterfaces();
@@ -183,7 +185,7 @@ int ConnectionManager::testConnections(bool background) {
 
         LOG_DEBUG(INFO, "Full reachability test started");
         ConnectionTester tester;
-        CHECK(tester.prepare(true /* full test */));
+        CHECK(tester.prepare(true /* full test */, lastTestFailed_));
         // Blocking call
         r = tester.runTest();
         LOG_DEBUG(INFO, "Full reachability test finished (%d)", r);
@@ -208,12 +210,20 @@ int ConnectionManager::testConnections(bool background) {
     }
     if (r == 0) {
         bestNetworks_.clear();
+        bool hasValidScore = false;
         for (auto& i: metrics) {
             bestNetworks_.append(std::make_pair(i.interface, i.resultingScore));
+            if (isValidScore(i.resultingScore)) {
+                hasValidScore = true;
+            }
         }
         if (background) {
             // Disable this for now
             // testResultsActual_ = true;
+        } else {
+            if (!hasValidScore) {
+                lastTestFailed_ = true;
+            }
         }
     }
     return r;
@@ -539,12 +549,27 @@ int ConnectionTester::pollSockets(struct pollfd* pfds, int socketCount) {
 // 3) Add these created+connected sockets to a pollfd structure. Allocate buffers for the reachability test messages.
 // 4) Poll all the sockets. Polling sends a reachability test message and waits for the response. The test continues for the test duration
 // 5) After polling completes, free the allocated buffers, reset diagnostics and calculate updated metrics. 
-int ConnectionTester::prepare(bool fullTest) {
+int ConnectionTester::prepare(bool fullTest, bool lastTestFailed) {
     struct addrinfo* info = nullptr;
     CloudServerAddressType type = CLOUD_SERVER_ADDRESS_TYPE_NONE;
 
     // Step 1: Retrieve the server hostname and port. Resolve the hostname to an addrinfo list (ie IP addresses of server)
-    CHECK(getCloudHostnameAndPort(&info, &type, !fullTest));
+    network_handle_t iface = NETWORK_INTERFACE_ALL;
+    if (lastTestFailed) {
+        for (int i = 0; i < metrics_.size(); i++) {
+            int idx = roundRobinInterfaceForDns_ % metrics_.size();
+            if (network_ready(metrics_[idx].interface, 0, nullptr)) {
+                iface = metrics_[idx].interface;
+                struct ifreq ifr = {};
+                if_index_to_name(iface, ifr.ifr_name);
+                LOG(TRACE, "Last rechability test failed, using interface %s for DNS queries explicitly", ifr.ifr_name);
+                roundRobinInterfaceForDns_++;
+                break;
+            }
+            roundRobinInterfaceForDns_++;
+        }
+    }
+    CHECK(getCloudHostnameAndPort(&info, &type, !fullTest, iface));
 
     SCOPE_GUARD({
         netdb_freeaddrinfo(info);
@@ -559,6 +584,24 @@ int ConnectionTester::prepare(bool fullTest) {
     // Step 2: Create, bind, and connect sockets for each network interface to test
     for (struct addrinfo* a = info; a != nullptr; a = a->ai_next) {
         bool ok = true;
+
+        switch (a->ai_family) {
+            case AF_INET: {
+                if (!network_ready(NETWORK_INTERFACE_ALL, NETWORK_READY_TYPE_IPV4, nullptr)) {
+                    LOG_DEBUG(WARN, "skipping resolved AF_INET address, as ipv4 networking is not ready");
+                    continue;
+                }
+                break;
+            }
+            case AF_INET6: {
+                if (!network_ready(NETWORK_INTERFACE_ALL, NETWORK_READY_TYPE_IPV6, nullptr)) {
+                    LOG_DEBUG(WARN, "skipping resolved AF_INET6 address, as ipv6 networking is not ready");
+                    continue;
+                }
+                break;
+            }
+        }
+
         // For each network interface to test, create + open a socket with the retrieved server address
         // If any of the sockets fail to be created + opened with this server address, return an error
         for (auto& connectionMetrics: metrics_) {