fabtests/efa: Improve MR exhaustion test

fabtests/prov/efa/src/efa_exhaust_mr_reg_common.c

@@ -2,23 +2,39 @@
 
 #include "efa_exhaust_mr_reg_common.h"
 
-int ft_efa_setup_ibv_pd(struct ibv_pd **pd)
-{
+int ft_efa_open_ibv_device(struct ibv_context **ctx) {
 	int num_dev = 0;
 	struct ibv_device **dev_list;
-	struct ibv_context *ctx;
 
 	dev_list = ibv_get_device_list(&num_dev);
 	if (num_dev < 1) {
 		FT_ERR("No ibv devices found");
 		ibv_free_device_list(dev_list);
-		return EXIT_FAILURE;
+		return -1;
 	} else if (num_dev > 1) {
 		FT_WARN("More than 1 ibv devices found! This test will only "
 			"exhaust MRs on the first device");
 	}
-	ctx = ibv_open_device(dev_list[0]);
+	*ctx = ibv_open_device(dev_list[0]);
 	ibv_free_device_list(dev_list);
+	return 0;
+}
+
+int ft_efa_close_ibv_device(struct ibv_context *ctx) {
+	return ibv_close_device(ctx);
+}
+
+int ft_efa_get_max_mr(struct ibv_context *ctx) {
+	int ret;
+	struct ibv_device_attr dev_attr = {0};
+	ret = ibv_query_device(ctx, &dev_attr);
+	if (ret)
+		FT_ERR("ibv_query_device failed with %d\n", ret);
+	return dev_attr.max_mr;
+}
+
+int ft_efa_setup_ibv_pd(struct ibv_context *ctx, struct ibv_pd **pd)
+{
 	*pd = ibv_alloc_pd(ctx);
 	if (!*pd) {
 		FT_ERR("alloc_pd failed with error %d\n", errno);
@@ -91,3 +107,48 @@ int ft_efa_alloc_bufs(void **buffers, size_t buf_size, size_t count) {
 	}
 	return FI_SUCCESS;
 }
+
+int ft_efa_unexpected_pingpong(void)
+{
+	int ret, i;
+
+	opts.options |= FT_OPT_OOB_CTRL;
+
+	ret = ft_sync();
+	if (ret)
+		return ret;
+
+	for (i = 0; i < opts.iterations + opts.warmup_iterations; i++) {
+		if (i == opts.warmup_iterations)
+			ft_start();
+
+		ret = ft_post_tx(ep, remote_fi_addr, opts.transfer_size, NO_CQ_DATA, &tx_ctx);
+		if (ret)
+			return ret;
+
+		ft_sync();
+
+		ret = ft_get_rx_comp(rx_seq);
+		if (ret)
+			return ret;
+
+		ret = ft_post_rx(ep, rx_size, &rx_ctx);
+		if (ret)
+			return ret;
+
+		ret = ft_get_tx_comp(tx_seq);
+		if (ret)
+			return ret;
+	}
+
+	ft_stop();
+
+	if (opts.machr)
+		show_perf_mr(opts.transfer_size, opts.iterations, &start, &end,
+			     2, opts.argc, opts.argv);
+	else
+		show_perf(NULL, opts.transfer_size, opts.iterations, &start,
+			  &end, 2);
+
+	return 0;
+}

fabtests/prov/efa/src/efa_exhaust_mr_reg_common.h

@@ -6,15 +6,17 @@
 #ifndef _EFA_EXHAUST_MR_REG_COMMON_H
 #define _EFA_EXHAUST_MR_REG_COMMON_H
 
-/* The EFA NIC currently supports a maximum of 256 * 1024 = 262144 registrations */
-#define EFA_MR_REG_LIMIT 262144
 #define EFA_MR_REG_BUF_SIZE 128
 
-int ft_efa_setup_ibv_pd(struct ibv_pd **pd);
+int ft_efa_open_ibv_device(struct ibv_context **ctx);
+int ft_efa_close_ibv_device(struct ibv_context *ctx);
+int ft_efa_get_max_mr(struct ibv_context *ctx);
+int ft_efa_setup_ibv_pd(struct ibv_context *ctx, struct ibv_pd **pd);
 void ft_efa_destroy_ibv_pd(struct ibv_pd *pd);
 int ft_efa_register_mr_reg(struct ibv_pd *pd, void **buffers, size_t buf_size,
 			   struct ibv_mr **mr_reg_vec, size_t count, size_t *registered);
 int ft_efa_deregister_mr_reg(struct ibv_mr **mr_reg_vec, size_t count);
 int ft_efa_alloc_bufs(void **buffers, size_t buf_size, size_t count);
+int ft_efa_unexpected_pingpong(void);
 
 #endif /* _EFA_EXHAUST_MR_REG_COMMON_H */

fabtests/prov/efa/src/efa_exhaust_mr_reg_rdm_pingpong.c

@@ -10,7 +10,7 @@
 #include "benchmarks/benchmark_shared.h"
 #include "efa_exhaust_mr_reg_common.h"
 
-static int run(void)
+static int run(int (*pingpong_func)(void))
 {
 	int i, ret = 0;
 
@@ -20,26 +20,27 @@ static int run(void)
 				continue;
 			opts.transfer_size = test_size[i].size;
 			init_test(&opts, test_name, sizeof(test_name));
-			ret = pingpong();
+			ret = pingpong_func();
 			if (ret)
 				return ret;
 		}
 	} else {
 		init_test(&opts, test_name, sizeof(test_name));
-		ret = pingpong();
+		ret = pingpong_func();
 		if (ret)
 			return ret;
 	}
 
-	return ft_finalize();
+	return 0;
 }
 
 int main(int argc, char **argv)
 {
-	int op, ret, err;
+	int op, ret, err, mr_reg_limit;
 	size_t registered;
-	void *buffers[EFA_MR_REG_LIMIT];
-	struct ibv_mr *mr_reg_vec[EFA_MR_REG_LIMIT];
+	void **buffers = NULL;
+	struct ibv_mr **mr_reg_vec = NULL;
+	struct ibv_context *ibv_ctx = NULL;
 	struct ibv_pd *pd;
 
 	opts = INIT_OPTS;
@@ -91,35 +92,56 @@ int main(int argc, char **argv)
 
 	ft_sync();
 	if (opts.dst_addr) {
-		err = ft_efa_alloc_bufs(buffers, EFA_MR_REG_BUF_SIZE,
-					EFA_MR_REG_LIMIT);
-		if (err)
-			FT_PRINTERR("alloc bufs", -err);
+		ret = ft_efa_open_ibv_device(&ibv_ctx);
+		if (ret)
+			FT_PRINTERR("ibv_open_device", -1);
 
-		err = ft_efa_setup_ibv_pd(&pd);
+		mr_reg_limit = ft_efa_get_max_mr(ibv_ctx);
+		printf("Memory registration limit on device %d\n", mr_reg_limit);
+
+		buffers = malloc(sizeof(void *) * mr_reg_limit);
+		mr_reg_vec = malloc(sizeof(struct ibv_reg_mr *) * mr_reg_limit);
+
+		err = ft_efa_setup_ibv_pd(ibv_ctx, &pd);
 		if (err)
 			FT_PRINTERR("ibv protection domain", -err);
 
 		printf("Exhausting MRs on client\n");
+		err = ft_efa_alloc_bufs(buffers, EFA_MR_REG_BUF_SIZE,
+					mr_reg_limit);
+		if (err)
+			FT_PRINTERR("alloc bufs", -err);
+
 		err = ft_efa_register_mr_reg(pd, buffers, EFA_MR_REG_BUF_SIZE,
-					     mr_reg_vec, EFA_MR_REG_LIMIT,
+					     mr_reg_vec, mr_reg_limit,
 					     &registered);
 		if (err)
 			FT_PRINTERR("ibv mr reg", -err);
 	}
 
 	ft_sync();
 	printf("Running pingpong test\n");
-	ret = run();
+	ret = run(pingpong);
+	if (ret)
+		goto out;
+
+	printf("Running unexpected pingpong test\n");
+	ret = run(ft_efa_unexpected_pingpong);
 
+out:
 	if (opts.dst_addr) {
 		printf("Deregistering MRs on client\n");
 		err = ft_efa_deregister_mr_reg(mr_reg_vec, registered);
 		if (err)
 			FT_PRINTERR("ibv mr dereg", -err);
 		ft_efa_destroy_ibv_pd(pd);
+		ft_efa_close_ibv_device(ibv_ctx);
 	}
 
+	free(buffers);
+	free(mr_reg_vec);
+
+	ft_finalize();
 	ft_free_res();
 
 	return ft_exit_code(ret);