diff --git a/src/graph/executor/StorageAccessExecutor.h b/src/graph/executor/StorageAccessExecutor.h index d29d17dfd25..7d089592e06 100644 --- a/src/graph/executor/StorageAccessExecutor.h +++ b/src/graph/executor/StorageAccessExecutor.h @@ -133,8 +133,17 @@ class StorageAccessExecutor : public Executor { auto &hostLatency = resp.hostLatency(); for (size_t i = 0; i < hostLatency.size(); ++i) { auto &info = hostLatency[i]; + auto &response = resp.responses()[i]; stats.emplace(folly::stringPrintf("%s exec/total", std::get<0>(info).toString().c_str()), folly::stringPrintf("%d(us)/%d(us)", std::get<1>(info), std::get<2>(info))); + if (response.result_ref()->latency_detail_us_ref().has_value()) { + std::string storageDetail = "{"; + for (auto iter : (*response.result_ref()->latency_detail_us_ref())) { + storageDetail += folly::stringPrintf("%s:%d(us),", iter.first.data(), iter.second); + } + storageDetail += "}"; + stats.emplace("storage_detail", storageDetail); + } } } diff --git a/src/graph/executor/query/GetNeighborsExecutor.cpp b/src/graph/executor/query/GetNeighborsExecutor.cpp index bcef5147823..0db01924fb9 100644 --- a/src/graph/executor/query/GetNeighborsExecutor.cpp +++ b/src/graph/executor/query/GetNeighborsExecutor.cpp @@ -79,6 +79,14 @@ folly::Future GetNeighborsExecutor::execute() { folly::stringPrintf("%s exec/total/vertices", std::get<0>(info).toString().c_str()), folly::stringPrintf( "%d(us)/%d(us)/%lu,", std::get<1>(info), std::get<2>(info), size)); + if (result.result.latency_detail_us_ref().has_value()) { + std::string storageDetail = "{"; + for (auto iter : (*result.result.latency_detail_us_ref())) { + storageDetail += folly::stringPrintf("%s:%d(us),", iter.first.data(), iter.second); + } + storageDetail += "}"; + otherStats_.emplace("storage_detail", storageDetail); + } } return handleResponse(resp); }); diff --git a/src/graph/executor/query/IndexScanExecutor.cpp b/src/graph/executor/query/IndexScanExecutor.cpp index c85cf74ee7f..3af563d6101 100644 --- a/src/graph/executor/query/IndexScanExecutor.cpp +++ b/src/graph/executor/query/IndexScanExecutor.cpp @@ -46,6 +46,7 @@ folly::Future IndexScanExecutor::indexScan() { lookup->returnColumns()) .via(runner()) .thenValue([this](StorageRpcResponse &&rpcResp) { + addStats(rpcResp, otherStats_); return handleResp(std::move(rpcResp)); }); } diff --git a/src/storage/BaseProcessor.h b/src/storage/BaseProcessor.h index 24731d5add7..2ecff99e5a4 100644 --- a/src/storage/BaseProcessor.h +++ b/src/storage/BaseProcessor.h @@ -43,6 +43,9 @@ class BaseProcessor { } this->result_.set_latency_in_us(this->duration_.elapsedInUSec()); + if (!profileDetail_.empty()) { + this->result_.set_latency_detail_us(std::move(profileDetail_)); + } this->result_.set_failed_parts(this->codes_); this->resp_.set_result(std::move(this->result_)); this->promise_.setValue(std::move(this->resp_)); @@ -102,6 +105,14 @@ class BaseProcessor { const std::vector& props, WriteResult& wRet); + virtual void profileDetail(const std::string& name, int32_t latency) { + if (!profileDetail_.count(name)) { + profileDetail_[name] = latency; + } else { + profileDetail_[name] += latency; + } + } + protected: StorageEnv* env_{nullptr}; const ProcessorCounters* counters_; @@ -116,6 +127,8 @@ class BaseProcessor { int32_t callingNum_{0}; int32_t spaceVidLen_; bool isIntId_; + std::map profileDetail_; + std::mutex profileMut_; }; } // namespace storage diff --git a/src/storage/StorageFlags.cpp b/src/storage/StorageFlags.cpp index c027aeac3b6..609cd90505f 100644 --- a/src/storage/StorageFlags.cpp +++ b/src/storage/StorageFlags.cpp @@ -47,3 +47,4 @@ DEFINE_bool(query_concurrently, false, "whether to run query of each part concurrently, only lookup and " "go are supported"); +DEFINE_bool(profile_storage_detail, false, "Whether to profile storage plan detail"); diff --git a/src/storage/StorageFlags.h b/src/storage/StorageFlags.h index 734a7a71b04..6b489d4f43d 100644 --- a/src/storage/StorageFlags.h +++ b/src/storage/StorageFlags.h @@ -35,4 +35,6 @@ DECLARE_int32(max_edge_returned_per_vertex); DECLARE_bool(query_concurrently); +DECLARE_bool(profile_storage_detail); + #endif // STORAGE_STORAGEFLAGS_H_ diff --git a/src/storage/exec/AggregateNode.h b/src/storage/exec/AggregateNode.h index f90f224ff38..9935185bebb 100644 --- a/src/storage/exec/AggregateNode.h +++ b/src/storage/exec/AggregateNode.h @@ -35,13 +35,15 @@ struct PropStat { template class AggregateNode : public IterateNode { public: - using RelNode::execute; + using RelNode::doExecute; AggregateNode(RuntimeContext* context, IterateNode* upstream, EdgeContext* edgeContext) - : IterateNode(upstream), context_(context), edgeContext_(edgeContext) {} + : IterateNode(upstream), context_(context), edgeContext_(edgeContext) { + IterateNode::name_ = "AggregateNode"; + } - nebula::cpp2::ErrorCode execute(PartitionID partId, const T& input) override { - auto ret = RelNode::execute(partId, input); + nebula::cpp2::ErrorCode doExecute(PartitionID partId, const T& input) override { + auto ret = RelNode::doExecute(partId, input); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { return ret; } @@ -51,8 +53,8 @@ class AggregateNode : public IterateNode { return nebula::cpp2::ErrorCode::SUCCEEDED; } - nebula::cpp2::ErrorCode execute(PartitionID partId) override { - auto ret = RelNode::execute(partId); + nebula::cpp2::ErrorCode doExecute(PartitionID partId) override { + auto ret = RelNode::doExecute(partId); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { return ret; } diff --git a/src/storage/exec/DeDupNode.h b/src/storage/exec/DeDupNode.h index f764bce8359..72c5436d12d 100644 --- a/src/storage/exec/DeDupNode.h +++ b/src/storage/exec/DeDupNode.h @@ -18,13 +18,15 @@ namespace storage { template class DeDupNode : public IterateNode { public: - using RelNode::execute; + using RelNode::doExecute; explicit DeDupNode(nebula::DataSet* resultSet, const std::vector& pos) - : resultSet_(resultSet), pos_(pos) {} + : resultSet_(resultSet), pos_(pos) { + IterateNode::name_ = "DedupNode"; + } - nebula::cpp2::ErrorCode execute(PartitionID partId) override { - auto ret = RelNode::execute(partId); + nebula::cpp2::ErrorCode doExecute(PartitionID partId) override { + auto ret = RelNode::doExecute(partId); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { return ret; } diff --git a/src/storage/exec/EdgeNode.h b/src/storage/exec/EdgeNode.h index a63285b9820..d0e1159d29e 100644 --- a/src/storage/exec/EdgeNode.h +++ b/src/storage/exec/EdgeNode.h @@ -52,9 +52,12 @@ class EdgeNode : public IterateNode { schemas_ = &(schemaIter->second); ttl_ = QueryUtils::getEdgeTTLInfo(edgeContext_, std::abs(edgeType_)); edgeName_ = edgeContext_->edgeNames_[edgeType_]; + IterateNode::name_ = "EdgeNode"; } - EdgeNode(RuntimeContext* context, EdgeContext* ctx) : context_(context), edgeContext_(ctx) {} + EdgeNode(RuntimeContext* context, EdgeContext* ctx) : context_(context), edgeContext_(ctx) { + IterateNode::name_ = "EdgeNode"; + } RuntimeContext* context_; EdgeContext* edgeContext_; @@ -71,7 +74,7 @@ class EdgeNode : public IterateNode { // FetchEdgeNode is used to fetch a single edge class FetchEdgeNode final : public EdgeNode { public: - using RelNode::execute; + using RelNode::doExecute; FetchEdgeNode(RuntimeContext* context, EdgeContext* edgeContext, @@ -79,7 +82,9 @@ class FetchEdgeNode final : public EdgeNode { const std::vector* props, StorageExpressionContext* expCtx = nullptr, Expression* exp = nullptr) - : EdgeNode(context, edgeContext, edgeType, props, expCtx, exp) {} + : EdgeNode(context, edgeContext, edgeType, props, expCtx, exp) { + name_ = "FetchEdgeNode"; + } bool valid() const override { return valid_; } @@ -91,9 +96,9 @@ class FetchEdgeNode final : public EdgeNode { RowReader* reader() const override { return reader_.get(); } - nebula::cpp2::ErrorCode execute(PartitionID partId, const cpp2::EdgeKey& edgeKey) override { + nebula::cpp2::ErrorCode doExecute(PartitionID partId, const cpp2::EdgeKey& edgeKey) override { valid_ = false; - auto ret = RelNode::execute(partId, edgeKey); + auto ret = RelNode::doExecute(partId, edgeKey); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { return ret; } @@ -143,14 +148,16 @@ class FetchEdgeNode final : public EdgeNode { // srcId class SingleEdgeNode final : public EdgeNode { public: - using RelNode::execute; + using RelNode::doExecute; SingleEdgeNode(RuntimeContext* context, EdgeContext* edgeContext, EdgeType edgeType, const std::vector* props, StorageExpressionContext* expCtx = nullptr, Expression* exp = nullptr) - : EdgeNode(context, edgeContext, edgeType, props, expCtx, exp) {} + : EdgeNode(context, edgeContext, edgeType, props, expCtx, exp) { + name_ = "SingleEdgeNode"; + } SingleEdgeIterator* iter() { return iter_.get(); } @@ -164,8 +171,8 @@ class SingleEdgeNode final : public EdgeNode { RowReader* reader() const override { return iter_->reader(); } - nebula::cpp2::ErrorCode execute(PartitionID partId, const VertexID& vId) override { - auto ret = RelNode::execute(partId, vId); + nebula::cpp2::ErrorCode doExecute(PartitionID partId, const VertexID& vId) override { + auto ret = RelNode::doExecute(partId, vId); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { return ret; } diff --git a/src/storage/exec/FilterNode.h b/src/storage/exec/FilterNode.h index b1fbc5d06bb..8baa8812dbd 100644 --- a/src/storage/exec/FilterNode.h +++ b/src/storage/exec/FilterNode.h @@ -24,22 +24,24 @@ data, but not both. As for GetNeighbors, it will have filter that involves both tag and edge expression. In that case, FilterNode has a upstream of HashJoinNode, which will keep popping out edge data. All tage data has been put into ExpressionContext -before FilterNode is executed. By that means, it can check the filter of tag + +before FilterNode is doExecuted. By that means, it can check the filter of tag + edge. */ template class FilterNode : public IterateNode { public: - using RelNode::execute; + using RelNode::doExecute; FilterNode(RuntimeContext* context, IterateNode* upstream, StorageExpressionContext* expCtx = nullptr, Expression* exp = nullptr) - : IterateNode(upstream), context_(context), expCtx_(expCtx), filterExp_(exp) {} + : IterateNode(upstream), context_(context), expCtx_(expCtx), filterExp_(exp) { + IterateNode::name_ = "FilterNode"; + } - nebula::cpp2::ErrorCode execute(PartitionID partId, const T& vId) override { - auto ret = RelNode::execute(partId, vId); + nebula::cpp2::ErrorCode doExecute(PartitionID partId, const T& vId) override { + auto ret = RelNode::doExecute(partId, vId); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { return ret; } diff --git a/src/storage/exec/GetNeighborsNode.h b/src/storage/exec/GetNeighborsNode.h index 0dd9ab477bd..b3f0eab036f 100644 --- a/src/storage/exec/GetNeighborsNode.h +++ b/src/storage/exec/GetNeighborsNode.h @@ -22,7 +22,7 @@ namespace storage { // target cell of a row. class GetNeighborsNode : public QueryNode { public: - using RelNode::execute; + using RelNode::doExecute; GetNeighborsNode(RuntimeContext* context, IterateNode* hashJoinNode, @@ -35,10 +35,12 @@ class GetNeighborsNode : public QueryNode { upstream_(upstream), edgeContext_(edgeContext), resultDataSet_(resultDataSet), - limit_(limit) {} + limit_(limit) { + name_ = "GetNeighborsNode"; + } - nebula::cpp2::ErrorCode execute(PartitionID partId, const VertexID& vId) override { - auto ret = RelNode::execute(partId, vId); + nebula::cpp2::ErrorCode doExecute(PartitionID partId, const VertexID& vId) override { + auto ret = RelNode::doExecute(partId, vId); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { return ret; } diff --git a/src/storage/exec/GetPropNode.h b/src/storage/exec/GetPropNode.h index 275c6c02fa3..845c6889885 100644 --- a/src/storage/exec/GetPropNode.h +++ b/src/storage/exec/GetPropNode.h @@ -16,15 +16,17 @@ namespace storage { class GetTagPropNode : public QueryNode { public: - using RelNode::execute; + using RelNode::doExecute; explicit GetTagPropNode(RuntimeContext* context, std::vector tagNodes, nebula::DataSet* resultDataSet) - : context_(context), tagNodes_(std::move(tagNodes)), resultDataSet_(resultDataSet) {} + : context_(context), tagNodes_(std::move(tagNodes)), resultDataSet_(resultDataSet) { + name_ = "GetTagPropNode"; + } - nebula::cpp2::ErrorCode execute(PartitionID partId, const VertexID& vId) override { - auto ret = RelNode::execute(partId, vId); + nebula::cpp2::ErrorCode doExecute(PartitionID partId, const VertexID& vId) override { + auto ret = RelNode::doExecute(partId, vId); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { return ret; } @@ -80,15 +82,17 @@ class GetTagPropNode : public QueryNode { class GetEdgePropNode : public QueryNode { public: - using RelNode::execute; + using RelNode::doExecute; GetEdgePropNode(RuntimeContext* context, std::vector*> edgeNodes, nebula::DataSet* resultDataSet) - : context_(context), edgeNodes_(std::move(edgeNodes)), resultDataSet_(resultDataSet) {} + : context_(context), edgeNodes_(std::move(edgeNodes)), resultDataSet_(resultDataSet) { + QueryNode::name_ = "GetEdgePropNode"; + } - nebula::cpp2::ErrorCode execute(PartitionID partId, const cpp2::EdgeKey& edgeKey) override { - auto ret = RelNode::execute(partId, edgeKey); + nebula::cpp2::ErrorCode doExecute(PartitionID partId, const cpp2::EdgeKey& edgeKey) override { + auto ret = RelNode::doExecute(partId, edgeKey); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { return ret; } diff --git a/src/storage/exec/HashJoinNode.h b/src/storage/exec/HashJoinNode.h index 3d98d975235..468bb598181 100644 --- a/src/storage/exec/HashJoinNode.h +++ b/src/storage/exec/HashJoinNode.h @@ -23,7 +23,7 @@ namespace storage { // return a iterator of edges which can pass ttl check and ready to be read. class HashJoinNode : public IterateNode { public: - using RelNode::execute; + using RelNode::doExecute; HashJoinNode(RuntimeContext* context, const std::vector& tagNodes, @@ -38,10 +38,11 @@ class HashJoinNode : public IterateNode { edgeContext_(edgeContext), expCtx_(expCtx) { UNUSED(tagContext_); + IterateNode::name_ = "HashJoinNode"; } - nebula::cpp2::ErrorCode execute(PartitionID partId, const VertexID& vId) override { - auto ret = RelNode::execute(partId, vId); + nebula::cpp2::ErrorCode doExecute(PartitionID partId, const VertexID& vId) override { + auto ret = RelNode::doExecute(partId, vId); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { return ret; } diff --git a/src/storage/exec/IndexEdgeNode.h b/src/storage/exec/IndexEdgeNode.h index 4dd27149d51..2810fc45f75 100644 --- a/src/storage/exec/IndexEdgeNode.h +++ b/src/storage/exec/IndexEdgeNode.h @@ -16,7 +16,7 @@ namespace storage { template class IndexEdgeNode final : public RelNode { public: - using RelNode::execute; + using RelNode::doExecute; IndexEdgeNode(RuntimeContext* context, IndexScanNode* indexScanNode, @@ -25,10 +25,12 @@ class IndexEdgeNode final : public RelNode { : context_(context), indexScanNode_(indexScanNode), schemas_(schemas), - schemaName_(schemaName) {} + schemaName_(schemaName) { + RelNode::name_ = "IndexEdgeNode"; + } - nebula::cpp2::ErrorCode execute(PartitionID partId) override { - auto ret = RelNode::execute(partId); + nebula::cpp2::ErrorCode doExecute(PartitionID partId) override { + auto ret = RelNode::doExecute(partId); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { return ret; } diff --git a/src/storage/exec/IndexFilterNode.h b/src/storage/exec/IndexFilterNode.h index 53701d8cdf0..ef5901d14fd 100644 --- a/src/storage/exec/IndexFilterNode.h +++ b/src/storage/exec/IndexFilterNode.h @@ -20,7 +20,7 @@ namespace storage { template class IndexFilterNode final : public RelNode { public: - using RelNode::execute; + using RelNode::doExecute; // evalExprByIndex_ is true, all fileds in filter is in index. No need to read // data anymore. @@ -35,6 +35,7 @@ class IndexFilterNode final : public RelNode { filterExp_(exp), isEdge_(isEdge) { evalExprByIndex_ = true; + RelNode::name_ = "IndexFilterNode"; } // evalExprByIndex_ is false, some fileds in filter is out of index, which @@ -59,9 +60,9 @@ class IndexFilterNode final : public RelNode { isEdge_ = false; } - nebula::cpp2::ErrorCode execute(PartitionID partId) override { + nebula::cpp2::ErrorCode doExecute(PartitionID partId) override { data_.clear(); - auto ret = RelNode::execute(partId); + auto ret = RelNode::doExecute(partId); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { return ret; } diff --git a/src/storage/exec/IndexOutputNode.h b/src/storage/exec/IndexOutputNode.h index 50d25bd2131..c9ed8433ceb 100644 --- a/src/storage/exec/IndexOutputNode.h +++ b/src/storage/exec/IndexOutputNode.h @@ -19,7 +19,7 @@ namespace storage { template class IndexOutputNode final : public RelNode { public: - using RelNode::execute; + using RelNode::doExecute; enum class IndexResultType : int8_t { kEdgeFromIndexScan, @@ -44,11 +44,13 @@ class IndexOutputNode final : public RelNode { fields_(fields) { type_ = context_->isEdge() ? IndexResultType::kEdgeFromIndexScan : IndexResultType::kVertexFromIndexScan; + RelNode::name_ = "IndexOpuputNode"; } IndexOutputNode(nebula::DataSet* result, RuntimeContext* context, IndexEdgeNode* indexEdgeNode) : result_(result), context_(context), indexEdgeNode_(indexEdgeNode) { type_ = IndexResultType::kEdgeFromDataScan; + RelNode::name_ = "IndexOpuputNode"; } IndexOutputNode(nebula::DataSet* result, @@ -56,6 +58,7 @@ class IndexOutputNode final : public RelNode { IndexVertexNode* indexVertexNode) : result_(result), context_(context), indexVertexNode_(indexVertexNode) { type_ = IndexResultType::kVertexFromDataScan; + RelNode::name_ = "IndexOpuputNode"; } IndexOutputNode(nebula::DataSet* result, @@ -72,10 +75,11 @@ class IndexOutputNode final : public RelNode { type_ = context_->isEdge() ? IndexResultType::kEdgeFromDataFilter : IndexResultType::kVertexFromDataFilter; } + RelNode::name_ = "IndexOpuputNode"; } - nebula::cpp2::ErrorCode execute(PartitionID partId) override { - auto ret = RelNode::execute(partId); + nebula::cpp2::ErrorCode doExecute(PartitionID partId) override { + auto ret = RelNode::doExecute(partId); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { return ret; } diff --git a/src/storage/exec/IndexScanNode.h b/src/storage/exec/IndexScanNode.h index 62337b9c6b2..6ecce4e5d31 100644 --- a/src/storage/exec/IndexScanNode.h +++ b/src/storage/exec/IndexScanNode.h @@ -16,7 +16,7 @@ namespace storage { template class IndexScanNode : public RelNode { public: - using RelNode::execute; + using RelNode::doExecute; IndexScanNode(RuntimeContext* context, IndexID indexId, @@ -36,10 +36,11 @@ class IndexScanNode : public RelNode { break; } } + RelNode::name_ = "IndexScanNode"; } - nebula::cpp2::ErrorCode execute(PartitionID partId) override { - auto ret = RelNode::execute(partId); + nebula::cpp2::ErrorCode doExecute(PartitionID partId) override { + auto ret = RelNode::doExecute(partId); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { return ret; } diff --git a/src/storage/exec/IndexVertexNode.h b/src/storage/exec/IndexVertexNode.h index 417255715cd..9ea3f43dcbc 100644 --- a/src/storage/exec/IndexVertexNode.h +++ b/src/storage/exec/IndexVertexNode.h @@ -16,7 +16,7 @@ namespace storage { template class IndexVertexNode final : public RelNode { public: - using RelNode::execute; + using RelNode::doExecute; IndexVertexNode(RuntimeContext* context, IndexScanNode* indexScanNode, @@ -25,10 +25,12 @@ class IndexVertexNode final : public RelNode { : context_(context), indexScanNode_(indexScanNode), schemas_(schemas), - schemaName_(schemaName) {} + schemaName_(schemaName) { + RelNode::name_ = "IndexVertexNode"; + } - nebula::cpp2::ErrorCode execute(PartitionID partId) override { - auto ret = RelNode::execute(partId); + nebula::cpp2::ErrorCode doExecute(PartitionID partId) override { + auto ret = RelNode::doExecute(partId); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { return ret; } diff --git a/src/storage/exec/RelNode.h b/src/storage/exec/RelNode.h index af347af0077..f493352f23e 100644 --- a/src/storage/exec/RelNode.h +++ b/src/storage/exec/RelNode.h @@ -9,6 +9,7 @@ #include "common/base/Base.h" #include "common/context/ExpressionContext.h" +#include "common/time/Duration.h" #include "common/utils/NebulaKeyUtils.h" #include "storage/CommonUtils.h" #include "storage/context/StorageExpressionContext.h" @@ -35,6 +36,12 @@ class RelNode { public: virtual nebula::cpp2::ErrorCode execute(PartitionID partId, const T& input) { + duration_.resume(); + auto ret = doExecute(partId, input); + duration_.pause(); + return ret; + } + virtual nebula::cpp2::ErrorCode doExecute(PartitionID partId, const T& input) { for (auto* dependency : dependencies_) { auto ret = dependency->execute(partId, input); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -45,6 +52,12 @@ class RelNode { } virtual nebula::cpp2::ErrorCode execute(PartitionID partId) { + duration_.resume(); + auto ret = doExecute(partId); + duration_.pause(); + return ret; + } + virtual nebula::cpp2::ErrorCode doExecute(PartitionID partId) { for (auto* dependency : dependencies_) { auto ret = dependency->execute(partId); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -65,9 +78,10 @@ class RelNode { explicit RelNode(const std::string& name) : name_(name) {} - std::string name_; + std::string name_ = "RelNode"; std::vector*> dependencies_; bool hasDependents_ = false; + time::Duration duration_{true}; }; // QueryNode is the node which would read data from kvstore, it usually generate diff --git a/src/storage/exec/StoragePlan.h b/src/storage/exec/StoragePlan.h index 4d16c379499..35f66b8ba91 100644 --- a/src/storage/exec/StoragePlan.h +++ b/src/storage/exec/StoragePlan.h @@ -81,6 +81,7 @@ class StoragePlan { CHECK_LT(idx, nodes_.size()); return nodes_[idx].get(); } + const std::vector>>& getNodes() { return nodes_; } private: bool firstLoop_ = true; diff --git a/src/storage/exec/TagNode.h b/src/storage/exec/TagNode.h index 20ef21a78f8..cf8f161bed0 100644 --- a/src/storage/exec/TagNode.h +++ b/src/storage/exec/TagNode.h @@ -17,7 +17,7 @@ namespace storage { // TagNode will return a DataSet of specified props of tagId class TagNode final : public IterateNode { public: - using RelNode::execute; + using RelNode::doExecute; TagNode(RuntimeContext* context, TagContext* ctx, @@ -39,11 +39,12 @@ class TagNode final : public IterateNode { schemas_ = &(schemaIter->second); ttl_ = QueryUtils::getTagTTLInfo(tagContext_, tagId_); tagName_ = tagContext_->tagNames_[tagId_]; + name_ = "TagNode"; } - nebula::cpp2::ErrorCode execute(PartitionID partId, const VertexID& vId) override { + nebula::cpp2::ErrorCode doExecute(PartitionID partId, const VertexID& vId) override { valid_ = false; - auto ret = RelNode::execute(partId, vId); + auto ret = RelNode::doExecute(partId, vId); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { return ret; } diff --git a/src/storage/exec/UpdateNode.h b/src/storage/exec/UpdateNode.h index 05e56ab5cdc..7e3f6536db4 100644 --- a/src/storage/exec/UpdateNode.h +++ b/src/storage/exec/UpdateNode.h @@ -37,7 +37,9 @@ class UpdateNode : public RelNode { insertable_(insertable), depPropMap_(depPropMap), expCtx_(expCtx), - isEdge_(isEdge) {} + isEdge_(isEdge) { + RelNode::name_ = "UpdateNode"; + } nebula::cpp2::ErrorCode checkField(const meta::SchemaProviderIf::Field* field) { if (!field) { @@ -151,7 +153,7 @@ class UpdateNode : public RelNode { // Update records, write to kvstore class UpdateTagNode : public UpdateNode { public: - using RelNode::execute; + using RelNode::doExecute; UpdateTagNode(RuntimeContext* context, std::vector> indexes, @@ -165,9 +167,10 @@ class UpdateTagNode : public UpdateNode { context, indexes, updatedProps, filterNode, insertable, depPropMap, expCtx, false), tagContext_(tagContext) { tagId_ = context_->tagId_; + name_ = "UpdateTagNode"; } - nebula::cpp2::ErrorCode execute(PartitionID partId, const VertexID& vId) override { + nebula::cpp2::ErrorCode doExecute(PartitionID partId, const VertexID& vId) override { CHECK_NOTNULL(context_->env()->kvstore_); IndexCountWrapper wrapper(context_->env()); @@ -181,7 +184,7 @@ class UpdateTagNode : public UpdateNode { return nebula::cpp2::ErrorCode::E_DATA_CONFLICT_ERROR; } - auto ret = RelNode::execute(partId, vId); + auto ret = RelNode::doExecute(partId, vId); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { return ret; } @@ -428,7 +431,7 @@ class UpdateTagNode : public UpdateNode { // Update records, write to kvstore class UpdateEdgeNode : public UpdateNode { public: - using RelNode::execute; + using RelNode::doExecute; UpdateEdgeNode(RuntimeContext* context, std::vector> indexes, @@ -442,9 +445,10 @@ class UpdateEdgeNode : public UpdateNode { context, indexes, updatedProps, filterNode, insertable, depPropMap, expCtx, true), edgeContext_(edgeContext) { edgeType_ = context_->edgeType_; + name_ = "UpdateEdgeNode"; } - nebula::cpp2::ErrorCode execute(PartitionID partId, const cpp2::EdgeKey& edgeKey) override { + nebula::cpp2::ErrorCode doExecute(PartitionID partId, const cpp2::EdgeKey& edgeKey) override { CHECK_NOTNULL(context_->env()->kvstore_); auto ret = nebula::cpp2::ErrorCode::SUCCEEDED; IndexCountWrapper wrapper(context_->env()); @@ -466,7 +470,7 @@ class UpdateEdgeNode : public UpdateNode { } auto op = [&partId, &edgeKey, this]() -> folly::Optional { - this->exeResult_ = RelNode::execute(partId, edgeKey); + this->exeResult_ = RelNode::doExecute(partId, edgeKey); if (this->exeResult_ == nebula::cpp2::ErrorCode::SUCCEEDED) { if (*edgeKey.edge_type_ref() != this->edgeType_) { this->exeResult_ = nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND; diff --git a/src/storage/exec/UpdateResultNode.h b/src/storage/exec/UpdateResultNode.h index 0fd6f14fcc4..c1b6fbf0e8a 100644 --- a/src/storage/exec/UpdateResultNode.h +++ b/src/storage/exec/UpdateResultNode.h @@ -17,7 +17,7 @@ namespace storage { template class UpdateResNode : public RelNode { public: - using RelNode::execute; + using RelNode::doExecute; UpdateResNode(RuntimeContext* context, RelNode* updateNode, @@ -28,10 +28,12 @@ class UpdateResNode : public RelNode { updateNode_(updateNode), returnPropsExp_(returnPropsExp), expCtx_(expCtx), - result_(result) {} + result_(result) { + RelNode::name_ = "UpdateResNode"; + } - nebula::cpp2::ErrorCode execute(PartitionID partId, const T& vId) override { - auto ret = RelNode::execute(partId, vId); + nebula::cpp2::ErrorCode doExecute(PartitionID partId, const T& vId) override { + auto ret = RelNode::doExecute(partId, vId); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED && ret != nebula::cpp2::ErrorCode::E_FILTER_OUT) { return ret; } diff --git a/src/storage/index/LookupBaseProcessor-inl.h b/src/storage/index/LookupBaseProcessor-inl.h index 9980ddb30bc..19c39aa43d0 100644 --- a/src/storage/index/LookupBaseProcessor-inl.h +++ b/src/storage/index/LookupBaseProcessor-inl.h @@ -443,6 +443,14 @@ LookupBaseProcessor::buildPlanWithDataAndFilter(nebula::DataSet* resu return output; } } +template +void LookupBaseProcessor::profilePlan(StoragePlan& plan) { + auto& nodes = plan.getNodes(); + std::lock_guard lck(BaseProcessor::profileMut_); + for (auto& node : nodes) { + BaseProcessor::profileDetail(node->name_, node->duration_.elapsedInUSec()); + } +} } // namespace storage } // namespace nebula diff --git a/src/storage/index/LookupBaseProcessor.h b/src/storage/index/LookupBaseProcessor.h index 967b4752d7c..0a3e35ef6b7 100644 --- a/src/storage/index/LookupBaseProcessor.h +++ b/src/storage/index/LookupBaseProcessor.h @@ -68,6 +68,8 @@ class LookupBaseProcessor : public BaseProcessor { StorageExpressionContext* exprCtx, Expression* exp); + void profilePlan(StoragePlan& plan); + protected: GraphSpaceID spaceId_; std::unique_ptr planContext_; diff --git a/src/storage/index/LookupProcessor.cpp b/src/storage/index/LookupProcessor.cpp index 83c8e7afcaf..8210c1b4666 100644 --- a/src/storage/index/LookupProcessor.cpp +++ b/src/storage/index/LookupProcessor.cpp @@ -60,6 +60,9 @@ void LookupProcessor::runInSingleThread(const cpp2::LookupIndexRequest& req) { } } } + if (FLAGS_profile_storage_detail) { + profilePlan(plan.value()); + } onProcessFinished(); onFinished(); } @@ -109,6 +112,9 @@ folly::Future> LookupProcessor:: return std::make_pair(nebula::cpp2::ErrorCode::E_INDEX_NOT_FOUND, partId); } auto ret = plan.value().go(partId); + if (FLAGS_profile_storage_detail) { + profilePlan(plan.value()); + } return std::make_pair(ret, partId); }); } diff --git a/src/storage/mutate/UpdateEdgeProcessor.cpp b/src/storage/mutate/UpdateEdgeProcessor.cpp index af79246902a..1679b7c87cc 100644 --- a/src/storage/mutate/UpdateEdgeProcessor.cpp +++ b/src/storage/mutate/UpdateEdgeProcessor.cpp @@ -88,6 +88,9 @@ void UpdateEdgeProcessor::doProcess(const cpp2::UpdateEdgeRequest& req) { onProcessFinished(); } } else { + if (FLAGS_profile_storage_detail) { + profilePlan(plan); + } onProcessFinished(); } onFinished(); diff --git a/src/storage/mutate/UpdateEdgeProcessor.h b/src/storage/mutate/UpdateEdgeProcessor.h index 6976e0209d9..fbd54f69959 100644 --- a/src/storage/mutate/UpdateEdgeProcessor.h +++ b/src/storage/mutate/UpdateEdgeProcessor.h @@ -57,6 +57,12 @@ class UpdateEdgeProcessor // result.begin(), get); return result; return returnPropsExp_; } + void profilePlan(StoragePlan& plan) { + auto& nodes = plan.getNodes(); + for (auto& node : nodes) { + profileDetail(node->name_, node->duration_.elapsedInUSec()); + } + } private: std::unique_ptr context_; diff --git a/src/storage/mutate/UpdateVertexProcessor.cpp b/src/storage/mutate/UpdateVertexProcessor.cpp index c181ab0257e..1bb129b15fd 100644 --- a/src/storage/mutate/UpdateVertexProcessor.cpp +++ b/src/storage/mutate/UpdateVertexProcessor.cpp @@ -82,6 +82,7 @@ void UpdateVertexProcessor::doProcess(const cpp2::UpdateVertexRequest& req) { onProcessFinished(); } } else { + profilePlan(plan); onProcessFinished(); } onFinished(); diff --git a/src/storage/mutate/UpdateVertexProcessor.h b/src/storage/mutate/UpdateVertexProcessor.h index c74d26efa3b..e81165146b8 100644 --- a/src/storage/mutate/UpdateVertexProcessor.h +++ b/src/storage/mutate/UpdateVertexProcessor.h @@ -58,6 +58,12 @@ class UpdateVertexProcessor // result.begin(), get); return result; return returnPropsExp_; } + void profilePlan(StoragePlan& plan) { + auto& nodes = plan.getNodes(); + for (auto& node : nodes) { + profileDetail(node->name_, node->duration_.elapsedInUSec()); + } + } private: std::unique_ptr context_; diff --git a/src/storage/query/GetNeighborsProcessor.cpp b/src/storage/query/GetNeighborsProcessor.cpp index 21e695b0a0b..bfe9d7a63ed 100644 --- a/src/storage/query/GetNeighborsProcessor.cpp +++ b/src/storage/query/GetNeighborsProcessor.cpp @@ -100,6 +100,9 @@ void GetNeighborsProcessor::runInSingleThread(const cpp2::GetNeighborsRequest& r } } } + if (FLAGS_profile_storage_detail) { + profilePlan(plan); + } onProcessFinished(); onFinished(); } @@ -165,6 +168,9 @@ folly::Future> GetNeighborsProce return std::make_pair(ret, partId); } } + if (FLAGS_profile_storage_detail) { + profilePlan(plan); + } return std::make_pair(nebula::cpp2::ErrorCode::SUCCEEDED, partId); }); } @@ -447,5 +453,12 @@ nebula::cpp2::ErrorCode GetNeighborsProcessor::checkStatType( void GetNeighborsProcessor::onProcessFinished() { resp_.set_vertices(std::move(resultDataSet_)); } +void GetNeighborsProcessor::profilePlan(StoragePlan& plan) { + auto& nodes = plan.getNodes(); + std::lock_guard lck(BaseProcessor::profileMut_); + for (auto& node : nodes) { + profileDetail(node->name_, node->duration_.elapsedInUSec()); + } +} } // namespace storage } // namespace nebula diff --git a/src/storage/query/GetNeighborsProcessor.h b/src/storage/query/GetNeighborsProcessor.h index cb085722561..1a3deb7c2af 100644 --- a/src/storage/query/GetNeighborsProcessor.h +++ b/src/storage/query/GetNeighborsProcessor.h @@ -75,6 +75,7 @@ class GetNeighborsProcessor const std::vector& rows, int64_t limit, bool random); + void profilePlan(StoragePlan& plan); private: std::vector contexts_;