performance/multi-node: use fi_inject_write.

 - Use fi_write_inject until the inject threshold is exceeded.
 - This is to compare the performance of fi_inject_write to
 fi_write.
Signed-off-by: Evan Harvey <eharvey@lanl.gov>

performance/multi-node/rdm_one_sided.c

@@ -534,6 +534,85 @@ void *thread_fn(void *data)
 	return NULL;
 }
 
+void *thread_fn_inject(void *data)
+{
+	int i, j, peer;
+	int size;
+	ssize_t __attribute__((unused)) fi_rc;
+	struct per_thread_data *ptd;
+	struct per_iteration_data it;
+	uint64_t t_start = 0, t_end = 0;
+
+	it.data = data;
+	size = it.message_size;
+
+	if (it.thread_id >= tunables.threads)
+		return (void *)-EINVAL;
+
+	ptd = &thread_data[it.thread_id];
+	ptd->bytes_sent = 0;
+
+	ct_tbarrier(&ptd->tbar);
+
+	if (myid == 0) {
+		peer = 1;
+
+		for (i = 0; i < loop + skip; i++) {
+			if (i == skip) {  /* warm up loop */
+				t_start = get_time_usec();
+				ptd->bytes_sent = 0;
+			}
+
+			for (j = 0; j < window_size; j++) {
+				fi_rc = fi_inject_write(ptd->ep,
+							ptd->s_buf,
+							size,
+							ptd->fi_addrs[peer],
+							ptd->rbuf_descs[peer].addr,
+							ptd->rbuf_descs[peer].key);
+				assert(fi_rc==FI_SUCCESS);
+				ptd->bytes_sent += size;
+			}
+		}
+
+		fi_rc = fi_send(ptd->ep, ptd->s_buf, 4, NULL,
+				ptd->fi_addrs[peer],
+				NULL);
+		assert(!fi_rc);
+		wait_for_comp(ptd->scq, 1);
+
+		fi_rc = fi_recv(ptd->ep, ptd->s_buf, 4, NULL,
+				ptd->fi_addrs[peer],
+				NULL);
+		assert(!fi_rc);
+		wait_for_comp(ptd->rcq, 1);
+
+		t_end = get_time_usec();
+	} else if (myid == 1) {
+		peer = 0;
+
+		fi_rc = fi_recv(ptd->ep, ptd->s_buf, 4, NULL,
+				ptd->fi_addrs[peer],
+				NULL);
+		assert(!fi_rc);
+		wait_for_comp(ptd->rcq, 1);
+
+		fi_rc = fi_send(ptd->ep, ptd->s_buf, 4, NULL,
+				ptd->fi_addrs[peer],
+				NULL);
+		assert(!fi_rc);
+		wait_for_comp(ptd->scq, 1);
+	}
+
+	ct_tbarrier(&ptd->tbar);
+
+	ptd->latency = (t_end - t_start) / (double)(loop * window_size);
+	ptd->time_start = t_start;
+	ptd->time_end = t_end;
+
+	return NULL;
+}
+
 int main(int argc, char *argv[])
 {
 	int op, ret;
@@ -683,7 +762,8 @@ int main(int argc, char *argv[])
 		for (i = 0; i < tunables.threads; i++) {
 			iter_key.thread_id = i;
 			ret = pthread_create(&thread_data[i].thread, NULL,
-					thread_fn, iter_key.data);
+					size <= fi->tx_attr->inject_size ?
+			thread_fn_inject : thread_fn, iter_key.data);
 			if (ret != 0) {
 				printf("couldn't create thread %i\n", i);
 				pthread_exit(NULL); /* a more robust exit would be nice here */