Skip to content

Commit e32d08b

Browse files
lrbisonlrbison
committed
coll/tuned: Refactor logging, and add coll_tuned_verbose mca parameter
Previously the coll_tuned output stream didn't use opal_output_set_verbosity at all. This made it difficult to distinguish problems from noisy logging. Signed-off-by: Luke Robison <lrbison@amazon.com>
1 parent 2a78444 commit e32d08b

21 files changed

+224
-133
lines changed

Diff for: ‎ompi/mca/coll/tuned/coll_tuned.h

+2
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@
3030

3131
BEGIN_C_DECLS
3232

33+
#define COLL_TUNED_TRACING_VERBOSE 50
34+
3335
/* these are the same across all modules and are loaded at component query time */
3436
extern int ompi_coll_tuned_stream;
3537
extern int ompi_coll_tuned_priority;

Diff for: ‎ompi/mca/coll/tuned/coll_tuned_allgather_decision.c

+2-2
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ int ompi_coll_tuned_allgather_intra_do_this(const void *sbuf, size_t scount,
133133
mca_coll_base_module_t *module,
134134
int algorithm, int faninout, int segsize)
135135
{
136-
OPAL_OUTPUT((ompi_coll_tuned_stream,
136+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
137137
"coll:tuned:allgather_intra_do_this selected algorithm %d topo faninout %d segsize %d",
138138
algorithm, faninout, segsize));
139139
switch (algorithm) {
@@ -174,7 +174,7 @@ int ompi_coll_tuned_allgather_intra_do_this(const void *sbuf, size_t scount,
174174
rbuf, rcount, rdtype,
175175
comm, module);
176176
} /* switch */
177-
OPAL_OUTPUT((ompi_coll_tuned_stream,
177+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
178178
"coll:tuned:allgather_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
179179
algorithm, ompi_coll_tuned_forced_max_algorithms[ALLGATHER]));
180180
return (MPI_ERR_ARG);

Diff for: ‎ompi/mca/coll/tuned/coll_tuned_allgatherv_decision.c

+2-2
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ int ompi_coll_tuned_allgatherv_intra_do_this(const void *sbuf, size_t scount,
133133
int algorithm, int faninout,
134134
int segsize)
135135
{
136-
OPAL_OUTPUT((ompi_coll_tuned_stream,
136+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
137137
"coll:tuned:allgatherv_intra_do_this selected algorithm %d topo faninout %d segsize %d",
138138
algorithm, faninout, segsize));
139139

@@ -167,7 +167,7 @@ int ompi_coll_tuned_allgatherv_intra_do_this(const void *sbuf, size_t scount,
167167
rbuf, rcounts, rdispls, rdtype,
168168
comm, module);
169169
} /* switch */
170-
OPAL_OUTPUT((ompi_coll_tuned_stream,
170+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
171171
"coll:tuned:allgatherv_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
172172
algorithm, ompi_coll_tuned_forced_max_algorithms[ALLGATHERV]));
173173
return (MPI_ERR_ARG);

Diff for: ‎ompi/mca/coll/tuned/coll_tuned_allreduce_decision.c

+6-4
Original file line numberDiff line numberDiff line change
@@ -130,8 +130,9 @@ int ompi_coll_tuned_allreduce_intra_do_this(const void *sbuf, void *rbuf, size_t
130130
mca_coll_base_module_t *module,
131131
int algorithm, int faninout, int segsize)
132132
{
133-
OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:allreduce_intra_do_this algorithm %d topo fan in/out %d segsize %d",
134-
algorithm, faninout, segsize));
133+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
134+
"coll:tuned:allreduce_intra_do_this algorithm %d topo fan in/out %d segsize %d",
135+
algorithm, faninout, segsize));
135136

136137
switch (algorithm) {
137138
case (0):
@@ -151,7 +152,8 @@ int ompi_coll_tuned_allreduce_intra_do_this(const void *sbuf, void *rbuf, size_t
151152
case (7):
152153
return ompi_coll_base_allreduce_intra_allgather_reduce(sbuf, rbuf, count, dtype, op, comm, module);
153154
} /* switch */
154-
OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:allreduce_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
155-
algorithm, ompi_coll_tuned_forced_max_algorithms[ALLREDUCE]));
155+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
156+
"coll:tuned:allreduce_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
157+
algorithm, ompi_coll_tuned_forced_max_algorithms[ALLREDUCE]));
156158
return (MPI_ERR_ARG);
157159
}

Diff for: ‎ompi/mca/coll/tuned/coll_tuned_alltoall_decision.c

+6-4
Original file line numberDiff line numberDiff line change
@@ -165,8 +165,9 @@ int ompi_coll_tuned_alltoall_intra_do_this(const void *sbuf, size_t scount,
165165
int algorithm, int faninout, int segsize,
166166
int max_requests)
167167
{
168-
OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:alltoall_intra_do_this selected algorithm %d topo faninout %d segsize %d",
169-
algorithm, faninout, segsize));
168+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
169+
"coll:tuned:alltoall_intra_do_this selected algorithm %d topo faninout %d segsize %d",
170+
algorithm, faninout, segsize));
170171

171172
switch (algorithm) {
172173
case (0):
@@ -182,7 +183,8 @@ int ompi_coll_tuned_alltoall_intra_do_this(const void *sbuf, size_t scount,
182183
case (5):
183184
return ompi_coll_base_alltoall_intra_two_procs(sbuf, scount, sdtype, rbuf, rcount, rdtype, comm, module);
184185
} /* switch */
185-
OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:alltoall_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
186-
algorithm, ompi_coll_tuned_forced_max_algorithms[ALLTOALL]));
186+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
187+
"coll:tuned:alltoall_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
188+
algorithm, ompi_coll_tuned_forced_max_algorithms[ALLTOALL]));
187189
return (MPI_ERR_ARG);
188190
}

Diff for: ‎ompi/mca/coll/tuned/coll_tuned_alltoallv_decision.c

+2-2
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ int ompi_coll_tuned_alltoallv_intra_do_this(const void *sbuf, ompi_count_array_t
9696
mca_coll_base_module_t *module,
9797
int algorithm)
9898
{
99-
OPAL_OUTPUT((ompi_coll_tuned_stream,
99+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
100100
"coll:tuned:alltoallv_intra_do_this selected algorithm %d ",
101101
algorithm));
102102

@@ -114,7 +114,7 @@ int ompi_coll_tuned_alltoallv_intra_do_this(const void *sbuf, ompi_count_array_t
114114
rbuf, rcounts, rdisps, rdtype,
115115
comm, module);
116116
} /* switch */
117-
OPAL_OUTPUT((ompi_coll_tuned_stream,
117+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
118118
"coll:tuned:alltoall_intra_do_this attempt to select "
119119
"algorithm %d when only 0-%d is valid.",
120120
algorithm, ompi_coll_tuned_forced_max_algorithms[ALLTOALLV]));

Diff for: ‎ompi/mca/coll/tuned/coll_tuned_barrier_decision.c

+4-3
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ int ompi_coll_tuned_barrier_intra_do_this (struct ompi_communicator_t *comm,
9191
mca_coll_base_module_t *module,
9292
int algorithm, int faninout, int segsize)
9393
{
94-
OPAL_OUTPUT((ompi_coll_tuned_stream,
94+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
9595
"coll:tuned:barrier_intra_do_this selected algorithm %d topo fanin/out%d",
9696
algorithm, faninout));
9797

@@ -104,7 +104,8 @@ int ompi_coll_tuned_barrier_intra_do_this (struct ompi_communicator_t *comm,
104104
case (5): return ompi_coll_base_barrier_intra_two_procs(comm, module);
105105
case (6): return ompi_coll_base_barrier_intra_tree(comm, module);
106106
} /* switch */
107-
OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:barrier_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
108-
algorithm, ompi_coll_tuned_forced_max_algorithms[BARRIER]));
107+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
108+
"coll:tuned:barrier_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
109+
algorithm, ompi_coll_tuned_forced_max_algorithms[BARRIER]));
109110
return (MPI_ERR_ARG);
110111
}

Diff for: ‎ompi/mca/coll/tuned/coll_tuned_bcast_decision.c

+6-4
Original file line numberDiff line numberDiff line change
@@ -140,8 +140,9 @@ int ompi_coll_tuned_bcast_intra_do_this(void *buf, size_t count,
140140
mca_coll_base_module_t *module,
141141
int algorithm, int faninout, int segsize)
142142
{
143-
OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:bcast_intra_do_this algorithm %d topo faninout %d segsize %d",
144-
algorithm, faninout, segsize));
143+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
144+
"coll:tuned:bcast_intra_do_this algorithm %d topo faninout %d segsize %d",
145+
algorithm, faninout, segsize));
145146

146147
switch (algorithm) {
147148
case (0):
@@ -166,7 +167,8 @@ int ompi_coll_tuned_bcast_intra_do_this(void *buf, size_t count,
166167
case (9):
167168
return ompi_coll_base_bcast_intra_scatter_allgather_ring(buf, count, dtype, root, comm, module, segsize);
168169
} /* switch */
169-
OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:bcast_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
170-
algorithm, ompi_coll_tuned_forced_max_algorithms[BCAST]));
170+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
171+
"coll:tuned:bcast_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
172+
algorithm, ompi_coll_tuned_forced_max_algorithms[BCAST]));
171173
return (MPI_ERR_ARG);
172174
}

Diff for: ‎ompi/mca/coll/tuned/coll_tuned_component.c

+25-10
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ char* ompi_coll_tuned_dynamic_rules_filename = (char*) NULL;
5656
int ompi_coll_tuned_init_tree_fanout = 4;
5757
int ompi_coll_tuned_init_chain_fanout = 4;
5858
int ompi_coll_tuned_init_max_requests = 128;
59+
int ompi_coll_tuned_verbose = 0;
5960

6061
/* Set it to the same value as intermediate msg by default, so it does not affect
6162
* default algorithm selection. Changing this value will force using linear with
@@ -191,6 +192,15 @@ static int tuned_register(void)
191192
MCA_BASE_VAR_SCOPE_ALL,
192193
&ompi_coll_tuned_dynamic_rules_filename);
193194

195+
ompi_coll_tuned_verbose = 0;
196+
(void) mca_base_component_var_register(&mca_coll_tuned_component.super.collm_version,
197+
"verbose",
198+
"Verbosity of the tuned coll component",
199+
MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
200+
OPAL_INFO_LVL_9,
201+
MCA_BASE_VAR_SCOPE_ALL,
202+
&ompi_coll_tuned_verbose);
203+
194204
/* register forced params */
195205
ompi_coll_tuned_allreduce_intra_check_forced_init(&ompi_coll_tuned_forced_params[ALLREDUCE]);
196206
ompi_coll_tuned_alltoall_intra_check_forced_init(&ompi_coll_tuned_forced_params[ALLTOALL]);
@@ -214,11 +224,10 @@ static int tuned_open(void)
214224
{
215225
int rc;
216226

217-
#if OPAL_ENABLE_DEBUG
218-
if (ompi_coll_base_framework.framework_verbose) {
227+
if (ompi_coll_tuned_verbose) {
219228
ompi_coll_tuned_stream = opal_output_open(NULL);
229+
opal_output_set_verbosity(ompi_coll_tuned_stream, ompi_coll_tuned_verbose);
220230
}
221-
#endif /* OPAL_ENABLE_DEBUG */
222231

223232
/* now check that the user hasn't overrode any of the decision functions if dynamic rules are enabled */
224233
/* the user can redo this before every comm dup/create if they like */
@@ -231,20 +240,24 @@ static int tuned_open(void)
231240
/* by default DISABLE dynamic rules and instead use fixed [if based] rules */
232241
if (ompi_coll_tuned_use_dynamic_rules) {
233242
if( ompi_coll_tuned_dynamic_rules_filename ) {
234-
OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:component_open Reading collective rules file [%s]",
235-
ompi_coll_tuned_dynamic_rules_filename));
243+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
244+
"coll:tuned:component_open Reading collective rules file [%s]",
245+
ompi_coll_tuned_dynamic_rules_filename));
236246
rc = ompi_coll_tuned_read_rules_config_file( ompi_coll_tuned_dynamic_rules_filename,
237247
&(mca_coll_tuned_component.all_base_rules));
238248
if( rc == OPAL_SUCCESS ) {
239-
OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:module_open Read a valid rules file"));
249+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
250+
"coll:tuned:module_open Read a valid rules file"));
240251
} else {
241-
OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:module_open Reading collective rules file failed\n"));
252+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
253+
"coll:tuned:module_open Reading collective rules file failed\n"));
242254
mca_coll_tuned_component.all_base_rules = NULL;
243255
}
244256
}
245257
}
246258

247-
OPAL_OUTPUT((ompi_coll_tuned_stream, "coll:tuned:component_open: done!"));
259+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
260+
"coll:tuned:component_open: done!"));
248261

249262
return OMPI_SUCCESS;
250263
}
@@ -253,12 +266,14 @@ static int tuned_open(void)
253266
/* i.e. alg table and dynamic changeable rules if allocated etc */
254267
static int tuned_close(void)
255268
{
256-
OPAL_OUTPUT((ompi_coll_tuned_stream, "coll:tuned:component_close: called"));
269+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
270+
"coll:tuned:component_close: called"));
257271

258272
/* dealloc alg table if allocated */
259273
/* dealloc dynamic changeable rules if allocated */
260274

261-
OPAL_OUTPUT((ompi_coll_tuned_stream, "coll:tuned:component_close: done!"));
275+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
276+
"coll:tuned:component_close: done!"));
262277

263278
if( NULL != mca_coll_tuned_component.all_base_rules ) {
264279
ompi_coll_tuned_free_all_rules(mca_coll_tuned_component.all_base_rules);

0 commit comments

Comments
 (0)