diff --git a/paddle/fluid/framework/ir/graph_helper.cc b/paddle/fluid/framework/ir/graph_helper.cc
index 97f486065ac62..b0a2b6754cb2a 100644
--- a/paddle/fluid/framework/ir/graph_helper.cc
+++ b/paddle/fluid/framework/ir/graph_helper.cc
@@ -579,6 +579,12 @@ void GraphToProgram(const Graph &graph,
 
     VLOG(3) << "Graph to program need convert " << graph.SubGraphsSize()
             << " sub graph";
+
+    std::unordered_set<std::string> vars_in_root_block;
+    for (const proto::VarDesc &var : block->vars()) {
+      vars_in_root_block.insert(var.name());
+    }
+
     for (size_t idx = 0; idx < graph.SubGraphsSize(); ++idx) {
       // avoid kRootBlockIndex not 0
       if (idx == kRootBlockIndex) continue;
@@ -586,7 +592,14 @@ void GraphToProgram(const Graph &graph,
       block = program_pb.add_blocks();
       block->set_idx(idx);
       block->set_parent_idx(kRootBlockIndex);
-      GraphToBlock(*graph.GetSubGraph(idx), block, sort_kind);
+
+      Graph *subgraph = graph.GetSubGraph(idx);
+      subgraph->SetNotOwned<std::unordered_set<std::string>>(
+          kGraphToProgramVarsToRemove, &vars_in_root_block);
+
+      GraphToBlock(*subgraph, block, sort_kind);
+
+      subgraph->Erase(kGraphToProgramVarsToRemove);
     }
   } else {
     GraphToBlock(graph, block, sort_kind);