Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

record memory and op supplement info #43550

Merged
merged 21 commits into from
Jun 24, 2022
Merged

record memory and op supplement info #43550

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
51f11b0
record memory and op supplement info
rainyfly Jun 15, 2022
06b21bb
update
rainyfly Jun 15, 2022
781fede
Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…
rainyfly Jun 15, 2022
315fc6b
update
rainyfly Jun 16, 2022
e3f305b
fix a bug
rainyfly Jun 16, 2022
6b3913e
fix memory recording
rainyfly Jun 17, 2022
d8048a0
fix a bug
rainyfly Jun 17, 2022
53e6a02
Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…
rainyfly Jun 17, 2022
10db9d5
update
rainyfly Jun 18, 2022
49edaf2
update
rainyfly Jun 18, 2022
04e5521
fix a bug
rainyfly Jun 19, 2022
2abf88d
update
rainyfly Jun 20, 2022
1c3aad0
fix a bug
rainyfly Jun 21, 2022
3c31453
fix a bug
rainyfly Jun 21, 2022
c1d4df5
fix a bug
rainyfly Jun 21, 2022
6bbf459
Revert "fix a bug"
rainyfly Jun 21, 2022
7e51f36
fix a bug
rainyfly Jun 21, 2022
2a37179
Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…
rainyfly Jun 21, 2022
ecfc073
fix format
rainyfly Jun 22, 2022
bdf718b
Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…
rainyfly Jun 23, 2022
335de82
fix
rainyfly Jun 23, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 70 additions & 38 deletions paddle/fluid/framework/new_executor/interpretercore.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,17 @@
#include "paddle/fluid/framework/operator.h"
#include "paddle/fluid/platform/os_info.h"
#include "paddle/fluid/platform/profiler/event_tracing.h"
#include "paddle/fluid/platform/profiler/supplement_tracing.h"
#include "paddle/phi/core/kernel_context.h"
#ifdef PADDLE_WITH_MKLDNN
#include "paddle/fluid/platform/mkldnn_helper.h"
#endif

PADDLE_DEFINE_EXPORTED_bool(new_executor_use_inplace, true,
PADDLE_DEFINE_EXPORTED_bool(new_executor_use_inplace,
true,
"Use inplace in new executor");
PADDLE_DEFINE_EXPORTED_bool(new_executor_use_local_scope, true,
PADDLE_DEFINE_EXPORTED_bool(new_executor_use_local_scope,
true,
"Use local_scope in new executor(especially used "
"in UT), can turn off for better performance");

Expand Down Expand Up @@ -172,12 +175,15 @@ paddle::framework::FetchList InterpreterCore::Run(
// scope?
}
global_scope_->SetLocalScope(local_scope_);
paddle::framework::interpreter::build_variable_scope(block_, global_scope_,
create_local_scope_);
paddle::framework::interpreter::build_variable_scope(
block_, global_scope_, create_local_scope_);
std::vector<paddle::framework::OpFuncNode> op_func_nodes;
paddle::framework::interpreter::build_op_func_list(
place_, block_, skip_gc_vars_, &op_func_nodes, global_scope_,
create_local_scope_);
paddle::framework::interpreter::build_op_func_list(place_,
block_,
skip_gc_vars_,
&op_func_nodes,
global_scope_,
create_local_scope_);
is_build_ = true;
SetFeedVarsInplaceSkip(feed_names);
// convert vec func_list to graph
Expand Down Expand Up @@ -505,7 +511,9 @@ void InterpreterCore::RunInstruction(const Instruction& instr_node) {
// If it is OperatorBase, InferShape do nothing.
if (op_with_kernel != nullptr) {
platform::RecordEvent infershape_event(
"infer_shape", platform::TracerEventType::OperatorInner, 1,
"infer_shape",
platform::TracerEventType::OperatorInner,
1,
platform::EventRole::kInnerOp);

// see OperatorWithKernel::RunImpl in operator.cc for why
Expand All @@ -514,6 +522,11 @@ void InterpreterCore::RunInstruction(const Instruction& instr_node) {
op_with_kernel->Info().infer_shape_(
instr_node.InnerInferShapeContext().get());
}
infershape_event.End();
platform::RecordOpInfoSupplement(op->Type(),
op->Attrs(),
*(instr_node.InnerInferShapeContext()),
*(instr_node.InnerRuntimeContext()));
}
}

Expand All @@ -531,7 +544,9 @@ void InterpreterCore::RunInstruction(const Instruction& instr_node) {

{
platform::RecordEvent compute_event(
"compute", platform::TracerEventType::OperatorInner, 1,
"compute",
platform::TracerEventType::OperatorInner,
1,
platform::EventRole::kInnerOp);
if (op_with_kernel == nullptr) {
instr_node.OpBase()->Run(*local_scope, place_);
Expand Down Expand Up @@ -588,7 +603,8 @@ void InterpreterCore::RunInstruction(const Instruction& instr_node) {
if (op_with_kernel != nullptr && FLAGS_check_nan_inf) {
VLOG(4) << "Check nan/inf";
framework::details::CheckOpHasNanOrInf(
*op, *global_scope_,
*op,
*global_scope_,
place); // TODO(xiongkun03) change it to inner scope.
}
}
Expand Down Expand Up @@ -617,10 +633,12 @@ void InterpreterCore::ExecuteInstructionList(
for (size_t i = 0; i < dependecy_count_.size(); ++i) {
if (dependecy_count_[i] == 0) {
async_work_queue_->AddTask(vec_instr.at(i).KernelType(),
[this, i, atomic_deps = atomic_deps.get(),
[this,
i,
atomic_deps = atomic_deps.get(),
atomic_var_ref = atomic_var_ref.get()] {
RunInstructionAsync(i, atomic_deps,
atomic_var_ref);
RunInstructionAsync(
i, atomic_deps, atomic_var_ref);
});
}
}
Expand All @@ -637,7 +655,8 @@ void InterpreterCore::ExecuteInstructionList(
}
VLOG(4) << "Cancel ok";
PADDLE_ENFORCE_EQ(
main_thread_blocker_.Clear(), 0,
main_thread_blocker_.Clear(),
0,
platform::errors::PreconditionNotMet(
"main_thread_blocker_.Clear() return -1, clear failed"));
VLOG(4) << "clear ok";
Expand All @@ -646,11 +665,12 @@ void InterpreterCore::ExecuteInstructionList(
}

void InterpreterCore::RunNextInstructions(
const Instruction& instr, std::queue<size_t>* reserved_next_ops,
const Instruction& instr,
std::queue<size_t>* reserved_next_ops,
std::vector<std::atomic<size_t>>* atomic_deps,
std::vector<std::atomic<size_t>>* atomic_var_ref) {
platform::RecordEvent record("RunNextInstructions",
platform::TracerEventType::UserDefined, 10);
platform::RecordEvent record(
"RunNextInstructions", platform::TracerEventType::UserDefined, 10);
VLOG(4) << "atomic 1:" << atomic_deps;
auto& next_instr = instr.NextInstructions();

Expand Down Expand Up @@ -716,7 +736,8 @@ void InterpreterCore::RunNextInstructions(
}

void InterpreterCore::RunInstructionAsync(
size_t instr_id, std::vector<std::atomic<size_t>>* atomic_deps,
size_t instr_id,
std::vector<std::atomic<size_t>>* atomic_deps,
std::vector<std::atomic<size_t>>* atomic_var_ref) {
std::queue<size_t> ready_ops;
ready_ops.push(instr_id);
Expand Down Expand Up @@ -787,8 +808,8 @@ void InterpreterCore::RecordStreamForGC(const Instruction& instr) {
instr.KernelType() != OpFuncType::kQueueAsync) {
return;
}
platform::RecordEvent record("RecordStreamForGC",
platform::TracerEventType::UserDefined, 10);
platform::RecordEvent record(
"RecordStreamForGC", platform::TracerEventType::UserDefined, 10);

gpuStream_t stream = reinterpret_cast<const platform::CUDADeviceContext&>(
instr.DeviceContext())
Expand Down Expand Up @@ -880,8 +901,8 @@ void InterpreterCore::RecordStreamForGC(const Instruction& instr) {
void InterpreterCore::CheckGC(
const Instruction& instr,
std::vector<std::atomic<size_t>>* atomic_var_ref) {
platform::RecordEvent record("CheckGC",
platform::TracerEventType::UserDefined, 10);
platform::RecordEvent record(
"CheckGC", platform::TracerEventType::UserDefined, 10);
size_t instr_id = instr.Id();
auto& var_scope = *global_scope_;

Expand All @@ -906,12 +927,14 @@ void InterpreterCore::CheckGC(

} else {
static_cast<InterpreterCoreEventGarbageCollector*>(gc_.get())->Add(
var_scope.Var(var_id), &gc_event_.at(instr_id),
var_scope.Var(var_id),
&gc_event_.at(instr_id),
&instr.DeviceContext());
}
#else
static_cast<InterpreterCoreEventGarbageCollector*>(gc_.get())->Add(
var_scope.Var(var_id), &gc_event_.at(instr_id),
var_scope.Var(var_id),
&gc_event_.at(instr_id),
&instr.DeviceContext());
#endif
}
Expand All @@ -920,20 +943,24 @@ void InterpreterCore::CheckGC(

void InterpreterCore::Prepare(
const std::vector<std::string>& feed_names,
const std::vector<framework::LoDTensor>& feed_tensors, bool prepare_feed) {
PADDLE_ENFORCE_EQ(feed_names.size(), feed_tensors.size(),
const std::vector<framework::LoDTensor>& feed_tensors,
bool prepare_feed) {
PADDLE_ENFORCE_EQ(feed_names.size(),
feed_tensors.size(),
platform::errors::PreconditionNotMet(
"Required feed_names.size() == feed_tensors.size(), "
"but received %d != %d",
feed_names.size(), feed_tensors.size()));
feed_names.size(),
feed_tensors.size()));

auto FeedInput = [&] {
VLOG(4) << "Feed inputs";
for (size_t i = 0; i < feed_names.size(); ++i) {
auto* feed_var = global_scope_->FindVar(feed_names[i]);
PADDLE_ENFORCE_NOT_NULL(
feed_var, platform::errors::NotFound(
"Variable %s should not be nullptr.", feed_names[i]));
feed_var,
platform::errors::NotFound("Variable %s should not be nullptr.",
feed_names[i]));

auto feed_tensor = feed_var->GetMutable<framework::LoDTensor>();
feed_tensor->ShareDataWith(feed_tensors[i]);
Expand All @@ -942,13 +969,16 @@ void InterpreterCore::Prepare(
};

if (!is_build_) {
paddle::framework::interpreter::build_variable_scope(block_, global_scope_,
create_local_scope_);
paddle::framework::interpreter::build_variable_scope(
block_, global_scope_, create_local_scope_);
FeedInput();
std::vector<paddle::framework::OpFuncNode> op_func_nodes;
paddle::framework::interpreter::build_op_func_list(
place_, block_, skip_gc_vars_, &op_func_nodes, global_scope_,
create_local_scope_);
paddle::framework::interpreter::build_op_func_list(place_,
block_,
skip_gc_vars_,
&op_func_nodes,
global_scope_,
create_local_scope_);
is_build_ = true;
SetFeedVarsInplaceSkip(feed_names);
// convert vec func_list to graph
Expand Down Expand Up @@ -1001,8 +1031,10 @@ void InterpreterCore::SetFeedVarsInplaceSkip(
}

std::shared_ptr<InterpreterCore> CreateInterpreterCore(
const platform::Place& place, const ProgramDesc& prog,
VariableScope* global_scope, const std::vector<std::string>& fetch_names,
const platform::Place& place,
const ProgramDesc& prog,
VariableScope* global_scope,
const std::vector<std::string>& fetch_names,
const std::set<std::string>& skip_gc_vars) {
std::shared_ptr<InterpreterCore> core = nullptr;
// NOTE(Aurelius84): `add_fetch` will modify BlockDesc, so we should copy
Expand All @@ -1011,8 +1043,8 @@ std::shared_ptr<InterpreterCore> CreateInterpreterCore(
auto* block = new_prog->MutableBlock(0);
interpreter::add_fetch(fetch_names, block);

core = std::make_shared<InterpreterCore>(place, *block, skip_gc_vars,
global_scope);
core = std::make_shared<InterpreterCore>(
place, *block, skip_gc_vars, global_scope);
core->SetCopyProgram(new_prog);
return core;
}
Expand Down
Loading