Skip to content

Commit 96dcdab

Browse files
sergachevGoogle-ML-Automation
authored andcommitted
PR #25166: Add slow operation alarm for argument initialization in the HLO runner.
Imported from GitHub PR #25166 Copybara import of the project: -- faa4fd1 by Ilia Sergachev <isergachev@nvidia.com>: Add slow operation alarm for argument initialization in the HLO runner. Merging this change closes #25166 FUTURE_COPYBARA_INTEGRATE_REVIEW=#25166 from openxla:add_slow_arg_init_alarm faa4fd1 PiperOrigin-RevId: 750111996
1 parent d05e130 commit 96dcdab

File tree

2 files changed

+8
-1
lines changed

2 files changed

+8
-1
lines changed

xla/tools/multihost_hlo_runner/BUILD

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,7 @@ cc_library(
168168
"//xla/service:hlo_module_config",
169169
"//xla/service:hlo_module_util",
170170
"//xla/service:hlo_proto_cc",
171+
"//xla/service:slow_operation_alarm",
171172
"//xla/tests:test_utils",
172173
"//xla/tools:hlo_control_flow_flattening",
173174
"//xla/tsl/platform:env",

xla/tools/multihost_hlo_runner/functional_hlo_runner.cc

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ limitations under the License.
6565
#include "xla/service/hlo.pb.h"
6666
#include "xla/service/hlo_module_config.h"
6767
#include "xla/service/hlo_module_util.h"
68+
#include "xla/service/slow_operation_alarm.h"
6869
#include "xla/shape_layout.h"
6970
#include "xla/shape_util.h"
7071
#include "xla/status_macros.h"
@@ -1161,6 +1162,11 @@ FunctionalHloRunner::CreateArgumentsOnDevice(
11611162
client, executable, running_options, flatten_arguments);
11621163
}
11631164

1165+
SlowOperationAlarm alarm(
1166+
absl::Seconds(5),
1167+
absl::StrFormat("Argument initialization is slow. Consider changing "
1168+
"--hlo_argument_mode."));
1169+
11641170
absl::Span<PjRtDevice* const> addressable_devices =
11651171
executable->addressable_devices();
11661172
size_t num_addressable_devices = addressable_devices.size();
@@ -1185,7 +1191,7 @@ FunctionalHloRunner::CreateArgumentsOnDevice(
11851191
ModuleArgumentMode::kUseZerosAsInput;
11861192

11871193
for (int i = 0; i < num_addressable_devices; ++i) {
1188-
VLOG(3) << "Creating fake argument for device " << i;
1194+
VLOG(3) << "Creating fake arguments for device " << i;
11891195
LiteralVec& argument_literals =
11901196
per_device_argument_literals[addressable_devices[i]->id()];
11911197
int executable_idx = hlo_modules.size() == 1

0 commit comments

Comments
 (0)