prov/efa: Allow disabling unsolicited write recv via env

Introduce FI_EFA_USE_UNSOLICITED_WRITE_RECV which means whether to use the unsolicited write recv feature from device when it's available. Setting this variable to 0 will disable this feature. Signed-off-by: Shi Jin <sjina@amazon.com>
shijin-aws · Jul 24, 2024 · 6927734 · 6927734
1 parent 4040785
commit 6927734
Show file tree

Hide file tree

Showing 7 changed files with 18 additions and 3 deletions.
diff --git a/man/fi_efa.7.md b/man/fi_efa.7.md
@@ -317,6 +317,10 @@ due to other requirements. See
 https://github.com/ofiwg/libfabric/blob/main/prov/efa/docs/efa_rdm_protocol_v4.md
 for details.
 
+*FI_EFA_USE_UNSOLICITED_WRITE_RECV*
+: Use device's unsolicited write recv functionality when it's available. (Default: 1).
+Setting this environment variable to 0 can disable this feature.
+
 # SEE ALSO
 
 [`fabric`(7)](fabric.7.html),

diff --git a/prov/efa/src/efa_base_ep.c b/prov/efa/src/efa_base_ep.c
@@ -181,7 +181,7 @@ int efa_qp_create(struct efa_qp **qp, struct ibv_qp_init_attr_ex *init_attr_ex)
 	} else {
 		assert(init_attr_ex->qp_type == IBV_QPT_DRIVER);
 #if HAVE_CAPS_UNSOLICITED_WRITE_RECV
-		if (efa_device_support_unsolicited_write_recv())
+		if (efa_rdm_use_unsolicited_write_recv())
 			efa_attr.flags |= EFADV_QP_FLAGS_UNSOLICITED_WRITE_RECV;
 #endif
 		efa_attr.driver_qp_type = EFADV_QP_DRIVER_TYPE_SRD;

diff --git a/prov/efa/src/efa_cq.h b/prov/efa/src/efa_cq.h
@@ -131,7 +131,7 @@ static inline int efa_cq_ibv_cq_ex_open(struct fi_cq_attr *attr,
 	};
 
 #if HAVE_CAPS_UNSOLICITED_WRITE_RECV
-	if (efa_device_support_unsolicited_write_recv())
+	if (efa_rdm_use_unsolicited_write_recv())
 		efadv_cq_init_attr.wc_flags |= EFADV_WC_EX_WITH_IS_UNSOLICITED;
 #endif
 

diff --git a/prov/efa/src/efa_env.c b/prov/efa/src/efa_env.c
@@ -37,6 +37,7 @@ struct efa_env efa_env = {
 	.host_id_file = "/sys/devices/virtual/dmi/id/board_asset_tag", /* Available on EC2 instances and containers */
 	.use_sm2 = false,
 	.huge_page_setting = EFA_ENV_HUGE_PAGE_UNSPEC,
+	.use_unsolicited_write_recv = 1,
 };
 
 /**
@@ -141,6 +142,7 @@ void efa_env_param_get(void)
 	fi_param_get_size_t(&efa_prov, "inter_max_gdrcopy_message_size",
 			    &efa_env.efa_max_gdrcopy_msg_size);
 	fi_param_get_bool(&efa_prov, "use_sm2", &efa_env.use_sm2);
+	fi_param_get_bool(&efa_prov, "use_unsolicited_write_recv", &efa_env.use_unsolicited_write_recv);
 
 	int use_huge_page;
 	if (fi_param_get_bool(&efa_prov, "use_huge_page", &use_huge_page) ==0) {
@@ -216,6 +218,8 @@ void efa_env_define()
 			"Using huge page memory has a small performance advantage, but can "
 			"cause system to run out of huge page memory. By default, EFA provider "
 			"will use huge page unless FI_EFA_FORK_SAFE is set to 1/on/true.");
+	fi_param_define(&efa_prov, "use_unsolicited_write_recv", FI_PARAM_BOOL,
+			"Use device's unsolicited write recv functionality when it's available. (Default: true)");
 }
 
 

diff --git a/prov/efa/src/efa_env.h b/prov/efa/src/efa_env.h
@@ -77,6 +77,7 @@ struct efa_env {
 	char *host_id_file;
 	int use_sm2;
 	enum efa_env_huge_page_setting huge_page_setting;
+	int use_unsolicited_write_recv;
 };
 
 /**

diff --git a/prov/efa/src/rdm/efa_rdm_cq.c b/prov/efa/src/rdm/efa_rdm_cq.c
@@ -83,7 +83,7 @@ static struct fi_ops efa_rdm_cq_fi_ops = {
 static inline
 bool efa_rdm_cq_wc_is_unsolicited(struct ibv_cq_ex *ibv_cq_ex)
 {
-	return efa_device_support_unsolicited_write_recv() ? efadv_wc_is_unsolicited(efadv_cq_from_ibv_cq_ex(ibv_cq_ex)) : false;
+	return efa_rdm_use_unsolicited_write_recv() ? efadv_wc_is_unsolicited(efadv_cq_from_ibv_cq_ex(ibv_cq_ex)) : false;
 }
 
 #else

diff --git a/prov/efa/src/rdm/efa_rdm_util.h b/prov/efa/src/rdm/efa_rdm_util.h
@@ -30,4 +30,10 @@ static inline void efa_rdm_poison_mem_region(void *ptr, size_t size)
 }
 #endif
 
+static inline
+bool efa_rdm_use_unsolicited_write_recv()
+{
+	return efa_env.use_unsolicited_write_recv && efa_device_support_unsolicited_write_recv();
+}
+
 #endif /* _EFA_RDM_UTIL_H */