From b37ab903183a34edf866c761cfb11a2bb84cec1d Mon Sep 17 00:00:00 2001 From: Xiaoli Zhou Date: Thu, 15 Jun 2023 11:54:33 +0800 Subject: [PATCH] [GIE Compiler] Add data type of algebra operator output to physical plan (#2870) Co-authored-by: Longbin Lai Co-authored-by: siyuan0322 --- .../ir/runtime/ffi/RelToFfiConverter.java | 57 +++ .../graphscope/common/jna/IrCoreLibrary.java | 14 + .../common/ir/runtime/FfiLogicalPlanTest.java | 16 +- .../test/resources/ffi_logical_plan_1.json | 328 +++++++++++------- .../test/resources/ffi_logical_plan_2.json | 46 ++- 5 files changed, 332 insertions(+), 129 deletions(-) diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java index d5396fd3d3a1..783153fe4229 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java @@ -85,6 +85,14 @@ public RelNode visit(GraphLogicalSource source) { if (source.getAliasId() != AliasInference.DEFAULT_ID) { checkFfiResult(LIB.setScanAlias(ptrScan, ArgUtils.asAlias(source.getAliasId()))); } + checkFfiResult( + LIB.setScanMeta( + ptrScan, + new FfiPbPointer.ByValue( + com.alibaba.graphscope.common.ir.runtime.proto.Utils.protoRowType( + source.getRowType(), isColumnId) + .get(0) + .toByteArray()))); return new PhysicalNode(source, ptrScan); } @@ -96,6 +104,14 @@ public RelNode visit(GraphLogicalExpand expand) { if (expand.getAliasId() != AliasInference.DEFAULT_ID) { checkFfiResult(LIB.setEdgexpdAlias(ptrExpand, ArgUtils.asAlias(expand.getAliasId()))); } + checkFfiResult( + LIB.setEdgexpdMeta( + ptrExpand, + new FfiPbPointer.ByValue( + com.alibaba.graphscope.common.ir.runtime.proto.Utils.protoRowType( + expand.getRowType(), isColumnId) + .get(0) + .toByteArray()))); return new PhysicalNode(expand, ptrExpand); } @@ -106,6 +122,14 @@ public RelNode visit(GraphLogicalGetV getV) { if (getV.getAliasId() != AliasInference.DEFAULT_ID) { checkFfiResult(LIB.setGetvAlias(ptrGetV, ArgUtils.asAlias(getV.getAliasId()))); } + checkFfiResult( + LIB.setGetvMeta( + ptrGetV, + new FfiPbPointer.ByValue( + com.alibaba.graphscope.common.ir.runtime.proto.Utils.protoRowType( + getV.getRowType(), isColumnId) + .get(0) + .toByteArray()))); return new PhysicalNode(getV, ptrGetV); } @@ -135,6 +159,15 @@ public RelNode visit(GraphLogicalSingleMatch match) { Pointer ptrSentence = LIB.initPatternSentence(FfiJoinKind.Inner); addFfiBinder(ptrSentence, match.getSentence(), true); checkFfiResult(LIB.addPatternSentence(ptrPattern, ptrSentence)); + com.alibaba.graphscope.common.ir.runtime.proto.Utils.protoRowType( + match.getRowType(), isColumnId) + .forEach( + k -> { + checkFfiResult( + LIB.addPatternMeta( + ptrPattern, + new FfiPbPointer.ByValue(k.toByteArray()))); + }); return new PhysicalNode(match, ptrPattern); case OPTIONAL: case ANTI: @@ -151,6 +184,14 @@ public RelNode visit(GraphLogicalMultiMatch match) { addFfiBinder(ptrSentence, sentence, true); checkFfiResult(LIB.addPatternSentence(ptrPattern, ptrSentence)); } + com.alibaba.graphscope.common.ir.runtime.proto.Utils.protoRowType( + match.getRowType(), isColumnId) + .forEach( + k -> { + checkFfiResult( + LIB.addPatternMeta( + ptrPattern, new FfiPbPointer.ByValue(k.toByteArray()))); + }); return new PhysicalNode(match, ptrPattern); } @@ -183,6 +224,14 @@ public PhysicalNode visit(GraphLogicalProject project) { new FfiPbPointer.ByValue(expression.toByteArray()), ffiAlias)); } + com.alibaba.graphscope.common.ir.runtime.proto.Utils.protoRowType( + project.getRowType(), isColumnId) + .forEach( + k -> { + checkFfiResult( + LIB.addProjectMeta( + ptrProject, new FfiPbPointer.ByValue(k.toByteArray()))); + }); return new PhysicalNode(project, ptrProject); } @@ -249,6 +298,14 @@ public PhysicalNode visit(GraphLogicalAggregate aggregate) { ffiAggOpt, ffiAlias)); } + com.alibaba.graphscope.common.ir.runtime.proto.Utils.protoRowType( + aggregate.getRowType(), isColumnId) + .forEach( + k -> { + checkFfiResult( + LIB.addGroupbyKeyValueMeta( + ptrGroup, new FfiPbPointer.ByValue(k.toByteArray()))); + }); return new PhysicalNode(aggregate, ptrGroup); } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/IrCoreLibrary.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/IrCoreLibrary.java index 842e2f236f34..f8a68b344323 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/IrCoreLibrary.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/IrCoreLibrary.java @@ -229,4 +229,18 @@ FfiResult.ByValue appendPatternOperator( FfiResult.ByValue addParamsExtra(Pointer params, String key, String value); Pointer initSinkGraphOperator(String graphName); + + FfiResult.ByValue setScanMeta(Pointer scan, FfiPbPointer.ByValue meta); + + FfiResult.ByValue setEdgexpdMeta(Pointer edgexpd, FfiPbPointer.ByValue meta); + + FfiResult.ByValue setGetvMeta(Pointer getV, FfiPbPointer.ByValue meta); + + FfiResult.ByValue addProjectMeta(Pointer project, FfiPbPointer.ByValue meta); + + FfiResult.ByValue addGroupbyKeyValueMeta(Pointer groupBy, FfiPbPointer.ByValue meta); + + FfiResult.ByValue addPatternMeta(Pointer pattern, FfiPbPointer.ByValue meta); + + FfiResult.ByValue setUnfoldMeta(Pointer unfold, FfiPbPointer.ByValue meta); } diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/FfiLogicalPlanTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/FfiLogicalPlanTest.java index 462c4ee874ab..389e9be64eab 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/FfiLogicalPlanTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/FfiLogicalPlanTest.java @@ -33,6 +33,7 @@ public class FfiLogicalPlanTest { // Match (x:person)-[:knows*1..3]->(:person {age: 10}) + // Return count(*) @Test public void logical_plan_1_test() throws Exception { GraphBuilder builder = Utils.mockGraphBuilder(); @@ -50,7 +51,7 @@ public void logical_plan_1_test() throws Exception { .pathOpt(GraphOpt.PathExpandPath.SIMPLE) .resultOpt(GraphOpt.PathExpandResult.ALL_V) .build(); - RelNode node = + RelNode aggregate = builder.source( new SourceConfig( GraphOpt.Source.VERTEX, @@ -63,26 +64,21 @@ public void logical_plan_1_test() throws Exception { GraphStdOperatorTable.EQUALS, pxdBuilder.variable(null, "age"), pxdBuilder.literal(10))) - .build(); - RelNode aggregate = - builder.match(node, GraphOpt.Match.INNER) .aggregate(builder.groupKey(), builder.count(builder.variable("x"))) .build(); Assert.assertEquals( "GraphLogicalAggregate(keys=[{variables=[], aliases=[]}], values=[[{operands=[x]," + " aggFunction=COUNT, alias='$f0', distinct=false}]])\n" - + " GraphLogicalSingleMatch(input=[null]," - + " sentence=[GraphLogicalGetV(tableConfig=[{isAll=false, tables=[person]}]," + + " GraphLogicalGetV(tableConfig=[{isAll=false, tables=[person]}]," + " alias=[DEFAULT], fusedFilter=[[=(DEFAULT.age, 10)]], opt=[END])\n" - + " GraphLogicalPathExpand(expand=[GraphLogicalExpand(tableConfig=[{isAll=false," + + " GraphLogicalPathExpand(expand=[GraphLogicalExpand(tableConfig=[{isAll=false," + " tables=[knows]}], alias=[DEFAULT], opt=[OUT])\n" + "], getV=[GraphLogicalGetV(tableConfig=[{isAll=false, tables=[person]}]," + " alias=[DEFAULT], opt=[END])\n" + "], offset=[1], fetch=[3], path_opt=[SIMPLE], result_opt=[ALL_V]," + " alias=[DEFAULT])\n" - + " GraphLogicalSource(tableConfig=[{isAll=false, tables=[person]}]," - + " alias=[x], opt=[VERTEX])\n" - + "], matchOpt=[INNER])", + + " GraphLogicalSource(tableConfig=[{isAll=false, tables=[person]}]," + + " alias=[x], opt=[VERTEX])", aggregate.explain().trim()); try (PhysicalBuilder ffiBuilder = new FfiPhysicalBuilder( diff --git a/interactive_engine/compiler/src/test/resources/ffi_logical_plan_1.json b/interactive_engine/compiler/src/test/resources/ffi_logical_plan_1.json index 9d69f1245a8d..3ada39d5992f 100644 --- a/interactive_engine/compiler/src/test/resources/ffi_logical_plan_1.json +++ b/interactive_engine/compiler/src/test/resources/ffi_logical_plan_1.json @@ -7,7 +7,7 @@ "scan_opt": 0, "alias": { "item": { - "Id": 1 + "Id": 0 } }, "params": { @@ -21,65 +21,56 @@ "columns": [], "is_all_columns": false, "limit": null, - "predicate": { - "operators": [ - { - "node_type": { - "type": { - "DataType": 1 - } - }, - "item": { - "Var": { - "tag": null, - "property": { - "item": { - "Key": { + "predicate": null, + "sample_ratio": 1.0, + "extra": {} + }, + "idx_predicate": null, + "meta_data": { + "type": { + "type": { + "GraphType": { + "element_opt": 0, + "graph_data_type": [ + { + "label": { + "label": 0, + "src_label": null, + "dst_label": null + }, + "props": [ + { + "prop_id": { + "item": { + "Name": "id" + } + }, + "type": 2 + }, + { + "prop_id": { + "item": { + "Name": "name" + } + }, + "type": 4 + }, + { + "prop_id": { "item": { "Name": "age" } - } - } - }, - "node_type": { - "type": { - "DataType": 1 + }, + "type": 1 } - } - } - } - }, - { - "node_type": { - "type": { - "DataType": 0 - } - }, - "item": { - "Logical": 0 - } - }, - { - "node_type": { - "type": { - "DataType": 1 + ] } - }, - "item": { - "Const": { - "item": { - "I32": 10 - } - } - } + ] } - ] + } }, - "sample_ratio": 1.0, - "extra": {} - }, - "idx_predicate": null, - "meta_data": null + "alias": 0 + } } } }, @@ -94,7 +85,7 @@ "base": { "edge_expand": { "v_tag": null, - "direction": 1, + "direction": 0, "params": { "tables": [ { @@ -112,7 +103,35 @@ }, "alias": null, "expand_opt": 1, - "meta_data": null + "meta_data": { + "type": { + "type": { + "GraphType": { + "element_opt": 1, + "graph_data_type": [ + { + "label": { + "label": 0, + "src_label": 0, + "dst_label": 0 + }, + "props": [ + { + "prop_id": { + "item": { + "Name": "weight" + } + }, + "type": 3 + } + ] + } + ] + } + } + }, + "alias": -1 + } }, "get_v": { "tag": null, @@ -133,14 +152,54 @@ "extra": {} }, "alias": null, - "meta_data": null - } - }, - "start_tag": { - "item": { - "Id": 1 + "meta_data": { + "type": { + "type": { + "GraphType": { + "element_opt": 0, + "graph_data_type": [ + { + "label": { + "label": 0, + "src_label": null, + "dst_label": null + }, + "props": [ + { + "prop_id": { + "item": { + "Name": "id" + } + }, + "type": 2 + }, + { + "prop_id": { + "item": { + "Name": "name" + } + }, + "type": 4 + }, + { + "prop_id": { + "item": { + "Name": "age" + } + }, + "type": 1 + } + ] + } + ] + } + } + }, + "alias": -1 + } } }, + "start_tag": null, "alias": null, "hop_range": { "lower": 1, @@ -163,42 +222,64 @@ "tag": null, "opt": 1, "params": { - "tables": [], + "tables": [ + { + "item": { + "Id": 0 + } + } + ], "columns": [], "is_all_columns": false, "limit": null, "predicate": { "operators": [ { - "node_type": null, + "node_type": { + "type": { + "DataType": 1 + } + }, "item": { "Var": { "tag": null, "property": { "item": { - "Label": {} + "Key": { + "item": { + "Name": "age" + } + } } }, - "node_type": null + "node_type": { + "type": { + "DataType": 1 + } + } } } }, { - "node_type": null, + "node_type": { + "type": { + "DataType": 0 + } + }, "item": { - "Logical": 6 + "Logical": 0 } }, { - "node_type": null, + "node_type": { + "type": { + "DataType": 1 + } + }, "item": { "Const": { "item": { - "I64Array": { - "item": [ - 0 - ] - } + "I32": 10 } } } @@ -208,57 +289,57 @@ "sample_ratio": 1.0, "extra": {} }, - "alias": { - "item": { - "Id": 0 - } - }, - "meta_data": null - } - } - }, - "children": [ - 3 - ] - }, - { - "opr": { - "opr": { - "Project": { - "mappings": [ - { - "expr": { - "operators": [ - { - "node_type": null, - "item": { - "Var": { - "tag": { - "item": { - "Id": 0 - } + "alias": null, + "meta_data": { + "type": { + "type": { + "GraphType": { + "element_opt": 0, + "graph_data_type": [ + { + "label": { + "label": 0, + "src_label": null, + "dst_label": null + }, + "props": [ + { + "prop_id": { + "item": { + "Name": "id" + } + }, + "type": 2 }, - "property": null, - "node_type": null - } + { + "prop_id": { + "item": { + "Name": "name" + } + }, + "type": 4 + }, + { + "prop_id": { + "item": { + "Name": "age" + } + }, + "type": 1 + } + ] } - } - ] - }, - "alias": { - "item": { - "Id": 0 + ] } } - } - ], - "is_append": false, - "meta_data": [] + }, + "alias": -1 + } } } }, "children": [ - 4 + 3 ] }, { @@ -328,12 +409,23 @@ } } ], - "meta_data": [] + "meta_data": [ + { + "type": { + "type": { + "DataType": 2 + } + }, + "alias": 1 + } + ] } } }, "children": [] } ], - "roots": [] + "roots": [ + 0 + ] } diff --git a/interactive_engine/compiler/src/test/resources/ffi_logical_plan_2.json b/interactive_engine/compiler/src/test/resources/ffi_logical_plan_2.json index f3c197fdddf2..4fb9a4194f2a 100644 --- a/interactive_engine/compiler/src/test/resources/ffi_logical_plan_2.json +++ b/interactive_engine/compiler/src/test/resources/ffi_logical_plan_2.json @@ -83,7 +83,51 @@ "extra": {} }, "idx_predicate": null, - "meta_data": null + "meta_data": { + "type": { + "type": { + "GraphType": { + "element_opt": 0, + "graph_data_type": [ + { + "label": { + "label": 0, + "src_label": null, + "dst_label": null + }, + "props": [ + { + "prop_id": { + "item": { + "Name": "id" + } + }, + "type": 2 + }, + { + "prop_id": { + "item": { + "Name": "name" + } + }, + "type": 4 + }, + { + "prop_id": { + "item": { + "Name": "age" + } + }, + "type": 1 + } + ] + } + ] + } + } + }, + "alias": 0 + } } } },