Merge pull request #948 from jsquyres/pr/v1.10/moar-usnic-updates

v1.10: More usnic updates

Author: rhc54

ompi/mca/btl/usnic/btl_usnic_endpoint.h

@@ -69,6 +69,7 @@ typedef struct opal_btl_usnic_modex_t {
     uint32_t ipv4_addr;
     /* Stored in host order */
     uint32_t ports[USNIC_NUM_CHANNELS];
+    /* Stored in network order */
     uint32_t netmask;
     /* Stored in host order */
     uint32_t connectivity_udp_port;

ompi/mca/btl/usnic/btl_usnic_module.c

@@ -98,11 +98,18 @@ static int add_procs_block_create_endpoints(opal_btl_usnic_module_t *module,
 
         /* Do not create loopback usnic connections */
         if (opal_proc == my_proc) {
+            opal_output_verbose(75, USNIC_OUT,
+                                "btl:usnic:add_procs:%s: not connecting to self",
+                                module->fabric_info->fabric_attr->name);
             continue;
         }
 
         /* usNIC does not support loopback to the same machine */
         if (OPAL_PROC_ON_LOCAL_NODE(opal_proc->proc_flags)) {
+            opal_output_verbose(75, USNIC_OUT,
+                                "btl:usnic:add_procs:%s: not connecting to %s on same server",
+                                module->fabric_info->fabric_attr->name,
+                                usnic_compat_proc_name_print(&opal_proc->proc_name));
             continue;
         }
 
@@ -115,6 +122,11 @@ static int add_procs_block_create_endpoints(opal_btl_usnic_module_t *module,
         if (OPAL_ERR_UNREACH == rc) {
             /* If the peer doesn't have usnic modex info, then we just
                skip it */
+            opal_output_verbose(75, USNIC_OUT,
+                                "btl:usnic:add_procs:%s: peer %s on %s does not have usnic modex info; skipping",
+                                module->fabric_info->fabric_attr->name,
+                                usnic_compat_proc_name_print(&opal_proc->proc_name),
+                                opal_get_proc_hostname(opal_proc));
             continue;
         } else if (OPAL_SUCCESS != rc) {
             return OPAL_ERR_OUT_OF_RESOURCE;
@@ -127,8 +139,10 @@ static int add_procs_block_create_endpoints(opal_btl_usnic_module_t *module,
                                             &usnic_endpoint);
         if (OPAL_SUCCESS != rc) {
             opal_output_verbose(5, USNIC_OUT,
-                                "btl:usnic:%s: unable to create endpoint for module=%p proc=%p\n",
-                                __func__, (void *)module, (void *)usnic_proc);
+                                "btl:usnic:add_procs:%s: unable to create endpoint to peer %s on %s",
+                                module->fabric_info->fabric_attr->name,
+                                usnic_compat_proc_name_print(&opal_proc->proc_name),
+                                opal_get_proc_hostname(opal_proc));
             OBJ_RELEASE(usnic_proc);
             continue;
         }
@@ -144,7 +158,8 @@ static int add_procs_block_create_endpoints(opal_btl_usnic_module_t *module,
                                           modex->netmask);
 
         opal_output_verbose(5, USNIC_OUT,
-                            "btl:usnic: new usnic peer endpoint: %s, proirity port %d, data port %d",
+                            "btl:usnic:add_procs:%s: new usnic peer endpoint: %s, proirity port %d, data port %d",
+                            module->fabric_info->fabric_attr->name,
                             str,
                             modex->ports[USNIC_PRIORITY_CHANNEL],
                             modex->ports[USNIC_DATA_CHANNEL]);
@@ -1895,6 +1910,7 @@ static void init_queue_lengths(opal_btl_usnic_module_t *module)
     } else {
         module->cq_num = mca_btl_usnic_component.cq_num;
     }
+    module->av_eq_num = mca_btl_usnic_component.av_eq_num;
 
     /*
      * Queue sizes for priority channel scale with # of endpoint. A

ompi/mca/btl/usnic/btl_usnic_util.c

@@ -115,24 +115,27 @@ opal_btl_usnic_dump_hex(void *vaddr, int len)
  * using inet_ntop()).
  */
 void opal_btl_usnic_snprintf_ipv4_addr(char *out, size_t maxlen,
-                                       uint32_t addr, uint32_t netmask)
+                                       uint32_t addr_be, uint32_t netmask_be)
 {
     int prefixlen;
+    uint32_t netmask = ntohl(netmask_be);
+    uint32_t addr = ntohl(addr_be);
     uint8_t *p = (uint8_t*) &addr;
+
     if (netmask != 0) {
         prefixlen = 33 - ffs(netmask);
         snprintf(out, maxlen, "%u.%u.%u.%u/%u",
-                 p[0],
-                 p[1],
-                 p[2],
                  p[3],
+                 p[2],
+                 p[1],
+                 p[0],
                  prefixlen);
     } else {
         snprintf(out, maxlen, "%u.%u.%u.%u",
-                 p[0],
-                 p[1],
+                 p[3],
                  p[2],
-                 p[3]);
+                 p[1],
+                 p[0]);
     }
 }
 

ompi/mca/btl/usnic/btl_usnic_util.h

@@ -113,7 +113,7 @@ void opal_btl_usnic_util_abort(const char *msg, const char *file, int line);
  * expected to be in network byte order.
  */
 void opal_btl_usnic_snprintf_ipv4_addr(char *out, size_t maxlen,
-                                       uint32_t addr, uint32_t netmask);
+                                       uint32_t addr_be, uint32_t netmask_be);
 
 void opal_btl_usnic_snprintf_bool_array(char *s, size_t slen, bool a[], size_t alen);