prov/efa: Use SHM's full inject size

Previously, we were subtracting EFA max header size from SHM inject size. Now that we use SHM as a peer provider, we can use SHM's full inject size of 4096. Also cleanup old deprecated env variable, and move hard coded value in place. Signed-off-by: Seth Zegelstein <szegel@amazon.com>
ofiwg · Nov 6, 2023 · 387a733 · 387a733
1 parent 7b5b580
commit 387a733
Show file tree

Hide file tree

Showing 5 changed files with 7 additions and 6 deletions.
diff --git a/prov/efa/src/efa.h b/prov/efa/src/efa.h
@@ -84,6 +84,8 @@
 
 #define EFA_ABI_VER_MAX_LEN 8
 
+#define SHM_MAX_INJECT_SIZE 4096
+
 #define EFA_EP_TYPE_IS_RDM(_info) \
 	(_info && _info->ep_attr && (_info->ep_attr->type == FI_EP_RDM))
 

diff --git a/prov/efa/src/efa_env.c b/prov/efa/src/efa_env.c
@@ -45,7 +45,6 @@ struct efa_env efa_env = {
 	.use_zcpy_rx = 1,
 	.zcpy_rx_seed = 0,
 	.shm_av_size = 256,
-	.shm_max_medium_size = 4096,
 	.recvwin_size = EFA_RDM_PEER_DEFAULT_REORDER_BUFFER_SIZE,
 	.ooo_pool_chunk_size = 64,
 	.unexp_pool_chunk_size = 1024,

diff --git a/prov/efa/src/efa_env.h b/prov/efa/src/efa_env.h
@@ -56,7 +56,6 @@ struct efa_env {
 	int zcpy_rx_seed;
 	int enable_shm_transfer;
 	int shm_av_size;
-	int shm_max_medium_size;
 	int recvwin_size;
 	int ooo_pool_chunk_size;
 	int unexp_pool_chunk_size;

diff --git a/prov/efa/src/efa_hmem.c b/prov/efa/src/efa_hmem.c
@@ -69,7 +69,7 @@ static int efa_domain_hmem_info_init_protocol_thresholds(struct efa_domain *efa_
 	case FI_HMEM_SYSTEM:
 		/* We have not yet tested runting with system memory */
 		info->runt_size = 0;
-		info->max_intra_eager_size = efa_env.shm_max_medium_size;
+		info->max_intra_eager_size = SHM_MAX_INJECT_SIZE;
 		info->max_medium_msg_size = EFA_DEFAULT_INTER_MAX_MEDIUM_MESSAGE_SIZE;
 		info->min_read_msg_size = EFA_DEFAULT_INTER_MIN_READ_MESSAGE_SIZE;
 		info->min_read_write_size = EFA_DEFAULT_INTER_MIN_READ_WRITE_SIZE;

diff --git a/prov/efa/src/efa_prov_info.c b/prov/efa/src/efa_prov_info.c
@@ -647,14 +647,15 @@ int efa_prov_info_alloc_for_rdm(struct fi_info **prov_info_rdm_ptr,
 		 *    pkt_entry_size - maximum_header_size.
 		 */
 		if (efa_env.enable_shm_transfer)
-			min_pkt_size = MIN(device->rdm_info->ep_attr->max_msg_size, efa_env.shm_max_medium_size);
+			min_pkt_size = MIN(device->rdm_info->ep_attr->max_msg_size - efa_rdm_pkt_type_get_max_hdr_size(),
+					   SHM_MAX_INJECT_SIZE);
 		else
-			min_pkt_size = device->rdm_info->ep_attr->max_msg_size;
+			min_pkt_size = device->rdm_info->ep_attr->max_msg_size - efa_rdm_pkt_type_get_max_hdr_size();
 
 		if (min_pkt_size < efa_rdm_pkt_type_get_max_hdr_size()) {
 			prov_info_rdm->tx_attr->inject_size = 0;
 		} else {
-			prov_info_rdm->tx_attr->inject_size = min_pkt_size - efa_rdm_pkt_type_get_max_hdr_size();
+			prov_info_rdm->tx_attr->inject_size = min_pkt_size;
 		}
 
 		/*