From 9d643e13553adc6272c3a09dfea9568081620391 Mon Sep 17 00:00:00 2001 From: Mikhail Babenko Date: Mon, 22 Jan 2024 16:01:11 +0300 Subject: [PATCH] fix ListUniqStable not working on optional arguments --- .../yql/core/common_opt/yql_co_simple1.cpp | 4 +-- .../yql/core/type_ann/type_ann_core.cpp | 4 ++- .../yql/core/type_ann/type_ann_list.cpp | 26 +++++++++---------- ydb/library/yql/core/type_ann/type_ann_list.h | 2 ++ .../sql/dq_file/part15/canondata/result.json | 6 ++--- .../tests/sql/sql2yql/canondata/result.json | 12 ++++----- .../yql/tests/sql/suites/expr/list_uniq.sql | 4 ++- .../part15/canondata/result.json | 12 ++++----- 8 files changed, 38 insertions(+), 32 deletions(-) diff --git a/ydb/library/yql/core/common_opt/yql_co_simple1.cpp b/ydb/library/yql/core/common_opt/yql_co_simple1.cpp index b53ec6b8011e..490a089f5c1e 100644 --- a/ydb/library/yql/core/common_opt/yql_co_simple1.cpp +++ b/ydb/library/yql/core/common_opt/yql_co_simple1.cpp @@ -3691,7 +3691,7 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) { map["Lookup"] = std::bind(&OptimizeContains, _1, _2); map["Contains"] = std::bind(&OptimizeContains, _1, _2); map["ListHas"] = std::bind(&OptimizeContains, _1, _2); - map["ListUniq"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext&) { + map["Uniq"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext&) { return ctx.Builder(node->Pos()) .Callable("DictKeys") .Callable(0, "ToDict") @@ -3713,7 +3713,7 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) { .Seal() .Build(); }; - map["ListUniqStable"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext&) { + map["UniqStable"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext&) { const TTypeAnnotationNode* itemType = node->Head().GetTypeAnn()->Cast()->GetItemType(); auto expandedItemType = ExpandType(node->Pos(), *itemType, ctx); auto setCreate = ctx.Builder(node->Pos()) diff --git a/ydb/library/yql/core/type_ann/type_ann_core.cpp b/ydb/library/yql/core/type_ann/type_ann_core.cpp index 61977e7cb5c6..5ad0621f1d9c 100644 --- a/ydb/library/yql/core/type_ann/type_ann_core.cpp +++ b/ydb/library/yql/core/type_ann/type_ann_core.cpp @@ -11907,6 +11907,8 @@ template Functions["AsSet"] = &AsDictWrapper; Functions["AsSetStrict"] = &AsDictWrapper; Functions["DictFromKeys"] = &DictFromKeysWrapper; + Functions["Uniq"] = &UniqWrapper; + Functions["UniqStable"] = &UniqWrapper; Functions["If"] = &IfWrapper; Functions["IfStrict"] = &IfWrapper; Functions[IfName] = &IfWorldWrapper; @@ -12170,7 +12172,7 @@ template Functions["ListNotNull"] = &ListNotNullWrapper; Functions["ListFlatten"] = &ListFlattenWrapper; Functions["ListUniq"] = &ListUniqWrapper; - Functions["ListUniqStable"] = &ListUniqWrapper; + Functions["ListUniqStable"] = &ListUniqStableWrapper; Functions["ExpandMap"] = &ExpandMapWrapper; Functions["WideMap"] = &WideMapWrapper; diff --git a/ydb/library/yql/core/type_ann/type_ann_list.cpp b/ydb/library/yql/core/type_ann/type_ann_list.cpp index 117881fcad50..ba3a0c4f78f9 100644 --- a/ydb/library/yql/core/type_ann/type_ann_list.cpp +++ b/ydb/library/yql/core/type_ann/type_ann_list.cpp @@ -7104,6 +7104,14 @@ namespace { } IGraphTransformer::TStatus ListUniqWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + return OptListWrapperImpl<1U>(input, output, ctx, "Uniq"); + } + + IGraphTransformer::TStatus ListUniqStableWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + return OptListWrapperImpl<1U>(input, output, ctx, "UniqStable"); + } + + IGraphTransformer::TStatus UniqWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { if (!EnsureArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -7112,27 +7120,19 @@ namespace { return IGraphTransformer::TStatus::Error; } - if (IsNull(input->Head())) { + auto type = input->Head().GetTypeAnn(); + + if (type->GetKind() == ETypeAnnotationKind::EmptyList) { output = input->HeadPtr(); return IGraphTransformer::TStatus::Repeat; } - auto type = input->Head().GetTypeAnn(); - if (type->GetKind() == ETypeAnnotationKind::Optional) { - type = type->Cast()->GetItemType(); - } - - if (type->GetKind() != ETypeAnnotationKind::List && type->GetKind() != ETypeAnnotationKind::EmptyList) { + if (type->GetKind() != ETypeAnnotationKind::List) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() - << "Expected (empty) list or optional of (empty) list, but got: " << *input->Head().GetTypeAnn())); + << "Expected (empty) list, but got: " << *input->Head().GetTypeAnn())); return IGraphTransformer::TStatus::Error; } - if (type->GetKind() == ETypeAnnotationKind::EmptyList) { - output = input->HeadPtr(); - return IGraphTransformer::TStatus::Repeat; - } - auto itemType = type->Cast()->GetItemType(); if (!itemType->IsHashable() || !itemType->IsEquatable()) { diff --git a/ydb/library/yql/core/type_ann/type_ann_list.h b/ydb/library/yql/core/type_ann/type_ann_list.h index 180626eba6e1..6b81f3ff53b5 100644 --- a/ydb/library/yql/core/type_ann/type_ann_list.h +++ b/ydb/library/yql/core/type_ann/type_ann_list.h @@ -129,6 +129,8 @@ namespace NTypeAnnImpl { IGraphTransformer::TStatus ListAllAnyWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus ListNotNullWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus ListUniqWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + IGraphTransformer::TStatus ListUniqStableWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + IGraphTransformer::TStatus UniqWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus ListFlattenWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus IterableWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus SqueezeToListWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); diff --git a/ydb/library/yql/tests/sql/dq_file/part15/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part15/canondata/result.json index c6716667cab3..9bb465260975 100644 --- a/ydb/library/yql/tests/sql/dq_file/part15/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part15/canondata/result.json @@ -848,9 +848,9 @@ ], "test.test[expr-list_uniq-default.txt-Debug]": [ { - "checksum": "8e6355d93491cc74a11c138ff795ea0d", - "size": 4028, - "uri": "https://{canondata_backend}/1600758/aad142702907f13e911494c1a7b312bad34f692a/resource.tar.gz#test.test_expr-list_uniq-default.txt-Debug_/opt.yql_patched" + "checksum": "321310a27b26dfda1c5cff67664f2247", + "size": 4110, + "uri": "https://{canondata_backend}/1600758/136704025b3d69d0c9001432d92d82db746f09f9/resource.tar.gz#test.test_expr-list_uniq-default.txt-Debug_/opt.yql_patched" } ], "test.test[expr-list_uniq-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/sql2yql/canondata/result.json b/ydb/library/yql/tests/sql/sql2yql/canondata/result.json index d2facbcd6eb0..e0d0d7eca1d0 100644 --- a/ydb/library/yql/tests/sql/sql2yql/canondata/result.json +++ b/ydb/library/yql/tests/sql/sql2yql/canondata/result.json @@ -5433,9 +5433,9 @@ ], "test_sql2yql.test[expr-list_uniq]": [ { - "checksum": "338a9dbb192084a632be4c818c35e5de", - "size": 2475, - "uri": "https://{canondata_backend}/1942671/8a805853ca0d5d0c7345fb32bb7b7e99685344f7/resource.tar.gz#test_sql2yql.test_expr-list_uniq_/sql.yql" + "checksum": "6a127889a51c049741d6929c6bc5eb42", + "size": 2917, + "uri": "https://{canondata_backend}/1942415/e8321dbd67e4e735ba3677524d3260f401d7c3c6/resource.tar.gz#test_sql2yql.test_expr-list_uniq_/sql.yql" } ], "test_sql2yql.test[expr-literal_list_element]": [ @@ -22800,9 +22800,9 @@ ], "test_sql_format.test[expr-list_uniq]": [ { - "checksum": "5c0e33f6414470dbe607fc1cb11bba4a", - "size": 283, - "uri": "https://{canondata_backend}/1880306/64654158d6bfb1289c66c626a8162239289559d0/resource.tar.gz#test_sql_format.test_expr-list_uniq_/formatted.sql" + "checksum": "74f6f47d443fa97917b2d52760056649", + "size": 375, + "uri": "https://{canondata_backend}/1942415/e8321dbd67e4e735ba3677524d3260f401d7c3c6/resource.tar.gz#test_sql_format.test_expr-list_uniq_/formatted.sql" } ], "test_sql_format.test[expr-literal_list_element]": [ diff --git a/ydb/library/yql/tests/sql/suites/expr/list_uniq.sql b/ydb/library/yql/tests/sql/suites/expr/list_uniq.sql index 8baab3f289dd..b27c5413d0a2 100644 --- a/ydb/library/yql/tests/sql/suites/expr/list_uniq.sql +++ b/ydb/library/yql/tests/sql/suites/expr/list_uniq.sql @@ -3,4 +3,6 @@ SELECT ListSort(ListUniq([1, 2, 1, 3, 4, 2, 4])), ListUniqStable([]), ListUniqStable([1, 2, 1, 3, 4, 2, 4]), ListUniqStable([1, 2, 3, null, 1, 7, 4, 3]), - ListUniqStable(["a", "b", "c", "a", "ab", "ac", "ab"]); \ No newline at end of file + ListUniqStable(["a", "b", "c", "a", "ab", "ac", "ab"]), + ListUniqStable(Just(["a", "b", "c", "a", "ab", "ac", "ab"])), + ListUniqStable(NULL); diff --git a/ydb/library/yql/tests/sql/yt_native_file/part15/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part15/canondata/result.json index 04eea3709e3d..284fd643b05e 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part15/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part15/canondata/result.json @@ -883,9 +883,9 @@ ], "test.test[expr-list_uniq-default.txt-Debug]": [ { - "checksum": "ea28d09a5802b3874193ccc6aa9388da", - "size": 3958, - "uri": "https://{canondata_backend}/1923547/61458c9c64b8429a1ff4c80acb29f295ac160173/resource.tar.gz#test.test_expr-list_uniq-default.txt-Debug_/opt.yql" + "checksum": "10364e75e35651100f741d926531468a", + "size": 4040, + "uri": "https://{canondata_backend}/1936842/8343e0a265567d3005787f49cdc980a3fffe72b5/resource.tar.gz#test.test_expr-list_uniq-default.txt-Debug_/opt.yql" } ], "test.test[expr-list_uniq-default.txt-Plan]": [ @@ -897,9 +897,9 @@ ], "test.test[expr-list_uniq-default.txt-Results]": [ { - "checksum": "a99901240a8c85fc59c830c24ae30cc3", - "size": 4624, - "uri": "https://{canondata_backend}/1923547/61458c9c64b8429a1ff4c80acb29f295ac160173/resource.tar.gz#test.test_expr-list_uniq-default.txt-Results_/results.txt" + "checksum": "1bd734251e39e8a92969dbd95ec5e9d1", + "size": 5714, + "uri": "https://{canondata_backend}/1936842/8343e0a265567d3005787f49cdc980a3fffe72b5/resource.tar.gz#test.test_expr-list_uniq-default.txt-Results_/results.txt" } ], "test.test[expr-to_hashed_set_dict_key-default.txt-Debug]": [