testsuite: test MPIR tool daemon launch

Problem: There are no tests of flux-job attach support for the MPIR
tool daemon launch extension.

Utilize the support in t/shell/mpir to drive testing of the code that
supports MPIR_executable_path and MPIR_server_arguments.

t/t2610-job-shell-mpir.t

@@ -48,5 +48,50 @@ test_expect_success 'flux-shell: test security of proctable method' '
 	flux job kill -s CONT ${id} &&
 	flux job attach ${id}
 '
+test_expect_success 'mpir: tool launch is supported' '
+	id=$(flux submit -N4 -n8 -o stop-tasks-in-exec /bin/true) &&
+	flux job wait-event -vt 5 -p exec -m sync=true $id shell.start &&
+	shell_rank=$(shell_leader_rank $id) &&
+	shell_service=$(shell_service $id) &&
+	${mpir} -r ${shell_rank} -s ${shell_service} \
+	    --tool-launch /bin/echo this is a tool >tool.log 2>&1 &&
+	test_debug "cat tool.log" &&
+	grep "MPIR: rank 0: echo: stdout: this is a tool" tool.log &&
+	grep "MPIR: rank 1: echo: stdout: this is a tool" tool.log &&
+	grep "MPIR: rank 2: echo: stdout: this is a tool" tool.log &&
+	grep "MPIR: rank 3: echo: stdout: this is a tool" tool.log &&
+	test $(grep -c "MPIR:.*this is a tool" tool.log) -eq 4 &&
+	flux job kill -s CONT $id &&
+	flux job attach $id
+'
+# Empty args for MPIR_executable_path -- also run with shell leader not
+# on broker rank 0 to ensure this configuration works with tool launch
+test_expect_success 'mpir: tool launch works with empty MPIR_server_arguments' '
+	id=$(flux submit --requires=rank:1,3 \
+	     -N2 -n2 -o stop-tasks-in-exec /bin/true) &&
+	flux job wait-event -vt 5 -p exec -m sync=true $id shell.start &&
+	shell_rank=$(shell_leader_rank $id) &&
+	shell_service=$(shell_service $id) &&
+	${mpir} -r ${shell_rank} -s ${shell_service} \
+	    --tool-launch hostname >tool2.log 2>&1 &&
+	test_debug "cat tool2.log" &&
+	grep "MPIR: rank 1: hostname: stdout:" tool2.log &&
+	grep "MPIR: rank 3: hostname: stdout:" tool2.log &&
+	test $(grep -c "MPIR:.*hostname:" tool2.log) -eq 2 &&
+	flux job kill -s CONT $id &&
+	flux job attach $id
+'
+test_expect_success 'mpir: tool launch reports errors' '
+	id=$(flux submit -N2 -n2 -o stop-tasks-in-exec /bin/true) &&
+	flux job wait-event -vt 5 -p exec -m sync=true $id shell.start &&
+	shell_rank=$(shell_leader_rank $id) &&
+	shell_service=$(shell_service $id) &&
+	${mpir} -r ${shell_rank} -s ${shell_service} \
+	    --tool-launch nosuchcommand >tool3.log 2>&1 &&
+	test_debug "cat tool3.log" &&
+	grep "MPIR: rank 0: nosuchcommand: error launching process" tool3.log &&
+	flux job kill -s CONT $id &&
+	flux job attach $id
+'
 
 test_done