diff --git a/.github/config/muted_ya.txt b/.github/config/muted_ya.txt index 9c96a8347eba..4e9d9e9464cc 100644 --- a/.github/config/muted_ya.txt +++ b/.github/config/muted_ya.txt @@ -6,15 +6,20 @@ ydb/core/blobstorage/ut_blobstorage VDiskAssimilation.Test ydb/core/blobstorage/ut_blobstorage [6/10]* ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk ReadOnlyVDisk.TestStorageLoad ydb/core/cms/ut_sentinel TSentinelTests.BSControllerCantChangeStatus +ydb/core/persqueue/ut [31/40]* ydb/core/persqueue/ut TPersQueueMirrorer.TestBasicRemote +ydb/core/quoter/ut QuoterWithKesusTest.PrefetchCoefficient ydb/core/kqp/ut/federated_query/generic * ydb/core/kqp/ut/scan KqpRequestContext.TraceIdInErrorMessage ydb/core/kqp/ut/service KqpQueryService.ExecuteQueryPgTableSelect ydb/core/tx/columnshard/ut_schema TColumnShardTestSchema.ForgetAfterFail ydb/core/tx/columnshard/ut_schema TColumnShardTestSchema.RebootForgetAfterFail +ydb/core/tx/columnshard/engines/ut * ydb/core/tx/coordinator/ut Coordinator.RestoreTenantConfiguration ydb/library/yql/providers/generic/connector/tests test.py.test_select_positive_postgresql* ydb/library/yql/sql/pg/ut PgSqlParsingAutoparam.AutoParamValues_DifferentTypes +ydb/library/yql/tests/sql/dq_file/part16 test.py.test[expr-as_dict_list_key-default.txt-Analyze] +ydb/library/yql/tests/sql/dq_file/part18 test.py.test[expr-cast_type_bind-default.txt-Analyze] ydb/public/sdk/cpp/client/ydb_topic/ut BasicUsage.WriteRead ydb/services/persqueue_v1/ut TPersQueueTest.DirectRead* ydb/services/ydb/sdk_sessions_pool_ut YdbSdkSessionsPool.StressTestSync10 @@ -23,13 +28,17 @@ ydb/services/ydb/ut YdbOlapStore.LogPagingAfter-NotNull ydb/tests/fq/s3 * ydb/tests/fq/yds test_metrics_cleanup.py.TestCleanup.test_cleanup[v1] ydb/tests/functional/audit * +ydb/tests/functional/blobstorage test_replication.py.TestReplicationAfterNodesRestart.test_replication[mirror-3-dc] ydb/tests/functional/clickbench test.py.test_plans* +ydb/tests/functional/clickbench test.py.test_run_determentistic[column] ydb/tests/functional/kqp/kqp_query_session KqpQuerySession.NoLocalAttach ydb/tests/functional/postgresql test_postgres.py.TestPostgresSuite.test_postgres_suite* ydb/tests/functional/restarts test_restarts.py.* ydb/tests/functional/sqs/cloud test_yandex_cloud_mode.py.TestSqsYandexCloudMode.test_dlq_mechanics_in_cloud* +ydb/tests/functional/sqs/cloud test_yandex_cloud_queue_counters.py.TestYmqQueueCounters.test_purge_queue_counters ydb/tests/functional/tenants test_dynamic_tenants.py.* ydb/tests/functional/tenants test_storage_config.py.TestStorageConfig.* ydb/tests/functional/tenants test_tenants.py.* ydb/tests/functional/ydb_cli test_ydb_scripting.py.TestScriptingServiceHelp.test_help ydb/tests/functional/ydb_cli test_ydb_scripting.py.TestScriptingServiceHelp.test_help_ex +ydb/tests/tools/pq_read/test test_timeout.py.TestTimeout.test_timeout diff --git a/ydb/core/keyvalue/keyvalue_ut_trace.cpp b/ydb/core/keyvalue/keyvalue_ut_trace.cpp new file mode 100644 index 000000000000..f918a189bdc8 --- /dev/null +++ b/ydb/core/keyvalue/keyvalue_ut_trace.cpp @@ -0,0 +1,154 @@ +#include +#include +#include +#include +#include +#include +#include + +#include + +using namespace NActors; +using namespace NKikimr; + +struct TTestEnvironment { + THolder Runtime; + const ui32 NodeCount; + TActorId Edge; + const ui64 TabletId = MakeTabletID(0, 0, 1); + const TTabletTypes::EType TabletType = TTabletTypes::KeyValue; + NWilson::TFakeWilsonUploader* WilsonUploader = nullptr; + + TTestEnvironment(ui32 nodeCount): NodeCount(nodeCount) { + } + + void Prepare() { + SetupRuntime(); + InitializeRuntime(); + + Edge = Runtime->AllocateEdgeActor(); + CreateTestBootstrapper(*Runtime, + CreateTestTabletInfo(TabletId, TabletType, TErasureType::ErasureNone), + &CreateKeyValueFlat); + SetupFakeWilson(); + + TDispatchOptions options; + options.FinalEvents.push_back(TDispatchOptions::TFinalEventCondition(TEvTablet::EvBoot)); + Runtime->DispatchEvents(options); + } + + void InitializeRuntime() { + TAppPrepare app; + app.AddDomain(TDomainsInfo::TDomain::ConstructEmptyDomain("dc-1").Release()); + SetupTabletServices(*Runtime, &app); + } + + void SetupRuntime() { + Runtime = MakeHolder(NodeCount, 1u); + + for (ui32 i = 0; i < NodeCount; ++i) { + SetupStateStorage(*Runtime, i, 0, true); + SetupTabletResolver(*Runtime, i); + } + } + + void SetupFakeWilson() { + WilsonUploader = new NWilson::TFakeWilsonUploader; + auto actorId = Runtime->Register(WilsonUploader); + Runtime->RegisterService(NWilson::MakeWilsonUploaderId(), actorId); + } + + template + auto DoKVRequest(THolder request) { + Runtime->SendToPipe(TabletId, Edge, request.Release(), 0, NTabletPipe::TClientConfig(), TActorId(), + 0, NWilson::TTraceId::NewTraceId(15, 4095)); + TAutoPtr handle; + auto response = Runtime->GrabEdgeEventRethrow(handle); + UNIT_ASSERT(response); + auto& record = response->Record; + UNIT_ASSERT_EQUAL(record.status(), NKikimrKeyValue::Statuses::RSTATUS_OK); + + return std::move(record); + } +}; + +THolder CreateWrite(TString key, TString value) { + auto request = MakeHolder(); + auto write = request->Record.add_commands()->mutable_write(); + write->set_key(std::move(key)); + write->set_value(std::move(value)); + return request; +} + +THolder CreateRead(TString key) { + auto request = MakeHolder(); + auto& record = request->Record; + record.set_key(std::move(key)); + record.set_offset(0); + record.set_size(0); + record.set_limit_bytes(0); + return request; +} + +void TestOneWrite(TString value, TString expectedTrace) { + TTestEnvironment env(8); + env.Prepare(); + + env.DoKVRequest(CreateWrite("key", std::move(value))); + + UNIT_ASSERT(env.WilsonUploader->BuildTraceTrees()); + UNIT_ASSERT_EQUAL(env.WilsonUploader->Traces.size(), 1); + auto& trace = env.WilsonUploader->Traces.begin()->second; + + UNIT_ASSERT_EQUAL(trace.ToString(), expectedTrace); +} + +void TestOneRead(TString value, TString expectedTrace) { + TTestEnvironment env(8); + env.Prepare(); + + env.DoKVRequest(CreateWrite("key", value)); + env.WilsonUploader->Clear(); + + auto response = env.DoKVRequest(CreateRead("key")); + UNIT_ASSERT_EQUAL(response.value(), value); + + UNIT_ASSERT(env.WilsonUploader->BuildTraceTrees()); + UNIT_ASSERT_EQUAL(env.WilsonUploader->Traces.size(), 1); + auto& trace = env.WilsonUploader->Traces.begin()->second; + + UNIT_ASSERT_EQUAL(trace.ToString(), expectedTrace); +} + +Y_UNIT_TEST_SUITE(TKeyValueTracingTest) { + const TString SmallValue = "value"; + const TString HugeValue = TString(1 << 20, 'v'); + +Y_UNIT_TEST(WriteSmall) { + TString canon = "(KeyValue.Intermediate -> [(KeyValue.StorageRequest -> [(DSProxy.Put -> [(Backpressure.InFlight " + "-> [(VDisk.Log.Put)])])]) , (Tablet.Transaction -> [(Tablet.Transaction.Execute) , (Tablet.WriteLog -> " + "[(Tablet.WriteLog.LogEntry -> [(DSProxy.Put -> [(Backpressure.InFlight -> [(VDisk.Log.Put)])])])])])])"; + TestOneWrite(SmallValue, std::move(canon)); +} + +Y_UNIT_TEST(WriteHuge) { + TString canon = "(KeyValue.Intermediate -> [(KeyValue.StorageRequest -> [(DSProxy.Put -> [(Backpressure.InFlight " + "-> [(VDisk.HugeBlobKeeper.Write -> [(VDisk.Log.PutHuge)])])])]) , (Tablet.Transaction -> " + "[(Tablet.Transaction.Execute) , (Tablet.WriteLog -> [(Tablet.WriteLog.LogEntry -> [(DSProxy.Put -> " + "[(Backpressure.InFlight -> [(VDisk.Log.Put)])])])])])])"; + TestOneWrite(HugeValue, std::move(canon)); +} + +Y_UNIT_TEST(ReadSmall) { + TString canon = "(KeyValue.Intermediate -> [(KeyValue.StorageReadRequest -> [(DSProxy.Get -> [(Backpressure.InFlight -> " + "[(VDisk.LevelIndexExtremeQueryViaBatcherMergeData)])])])])"; + TestOneRead(SmallValue, std::move(canon)); +} + +Y_UNIT_TEST(ReadHuge) { + TString canon = "(KeyValue.Intermediate -> [(KeyValue.StorageReadRequest -> [(DSProxy.Get -> [(Backpressure.InFlight -> " + "[(VDisk.LevelIndexExtremeQueryViaBatcherMergeData -> [(VDisk.Query.ReadBatcher)])])])])])"; + TestOneRead(HugeValue, std::move(canon)); +} + +} diff --git a/ydb/core/keyvalue/ut_trace/ya.make b/ydb/core/keyvalue/ut_trace/ya.make new file mode 100644 index 000000000000..a6c8e0b9d692 --- /dev/null +++ b/ydb/core/keyvalue/ut_trace/ya.make @@ -0,0 +1,26 @@ +UNITTEST_FOR(ydb/core/keyvalue) + +FORK_SUBTESTS() + +SPLIT_FACTOR(5) + +IF (SANITIZER_TYPE == "thread" OR WITH_VALGRIND) + TIMEOUT(1800) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +PEERDIR( + ydb/core/testlib/default +) + +SRCS( + keyvalue_ut_trace.cpp +) + +REQUIREMENTS(ram:16) + +END() diff --git a/ydb/core/keyvalue/ya.make b/ydb/core/keyvalue/ya.make index f8cfaab1866f..3a014e31316d 100644 --- a/ydb/core/keyvalue/ya.make +++ b/ydb/core/keyvalue/ya.make @@ -62,4 +62,5 @@ RECURSE( RECURSE_FOR_TESTS( ut + ut_trace ) diff --git a/ydb/core/kqp/executer_actor/kqp_executer_stats.cpp b/ydb/core/kqp/executer_actor/kqp_executer_stats.cpp index 6b8b194b27d0..34633516825a 100644 --- a/ydb/core/kqp/executer_actor/kqp_executer_stats.cpp +++ b/ydb/core/kqp/executer_actor/kqp_executer_stats.cpp @@ -330,7 +330,7 @@ bool CollectProfileStats(Ydb::Table::QueryStatsCollection::Mode statsMode) { void TQueryExecutionStats::FillStageDurationUs(NYql::NDqProto::TDqStageStats& stats) { if (stats.HasStartTimeMs() && stats.HasFinishTimeMs()) { auto startTimeMs = stats.GetStartTimeMs().GetMin(); - auto finishTimeMs = stats.GetFinishTimeMs().GetMin(); + auto finishTimeMs = stats.GetFinishTimeMs().GetMax(); if (startTimeMs && finishTimeMs > startTimeMs) { stats.SetStageDurationUs((finishTimeMs - startTimeMs) * 1'000); } diff --git a/ydb/core/kqp/ut/common/columnshard.cpp b/ydb/core/kqp/ut/common/columnshard.cpp index 212d49076d02..99a10e26ecad 100644 --- a/ydb/core/kqp/ut/common/columnshard.cpp +++ b/ydb/core/kqp/ut/common/columnshard.cpp @@ -9,7 +9,6 @@ namespace NKqp { TTestHelper::TTestHelper(const TKikimrSettings& settings) : Kikimr(settings) , TableClient(Kikimr.GetTableClient()) - , LongTxClient(Kikimr.GetDriver()) , Session(TableClient.CreateSession().GetValueSync().GetSession()) {} @@ -31,26 +30,6 @@ namespace NKqp { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); } - void TTestHelper::InsertData(const TColumnTable& table, TTestHelper::TUpdatesBuilder& updates, const std::function onBeforeCommit /*= {}*/, const EStatus opStatus /*= EStatus::SUCCESS*/) { - NLongTx::TLongTxBeginResult resBeginTx = LongTxClient.BeginWriteTx().GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resBeginTx.Status().GetStatus(), EStatus::SUCCESS, resBeginTx.Status().GetIssues().ToString()); - - auto txId = resBeginTx.GetResult().tx_id(); - auto batch = updates.BuildArrow(); - TString data = NArrow::NSerialization::TFullDataSerializer(arrow::ipc::IpcWriteOptions::Defaults()).Serialize(batch); - - NLongTx::TLongTxWriteResult resWrite = - LongTxClient.Write(txId, table.GetName(), txId, data, Ydb::LongTx::Data::APACHE_ARROW).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resWrite.Status().GetStatus(), opStatus, resWrite.Status().GetIssues().ToString()); - - if (onBeforeCommit) { - onBeforeCommit(); - } - - NLongTx::TLongTxCommitResult resCommitTx = LongTxClient.CommitTx(txId).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resCommitTx.Status().GetStatus(), EStatus::SUCCESS, resCommitTx.Status().GetIssues().ToString()); - } - void TTestHelper::BulkUpsert(const TColumnTable& table, TTestHelper::TUpdatesBuilder& updates, const Ydb::StatusIds_StatusCode& opStatus /*= Ydb::StatusIds::SUCCESS*/) { Y_UNUSED(opStatus); NKikimr::Tests::NCS::THelper helper(Kikimr.GetTestServer()); diff --git a/ydb/core/kqp/ut/common/columnshard.h b/ydb/core/kqp/ut/common/columnshard.h index 92b45eb60759..adc2ea16989a 100644 --- a/ydb/core/kqp/ut/common/columnshard.h +++ b/ydb/core/kqp/ut/common/columnshard.h @@ -3,7 +3,6 @@ #include "kqp_ut_common.h" #include #include -#include #include #include #include @@ -64,7 +63,6 @@ namespace NKqp { private: TKikimrRunner Kikimr; NYdb::NTable::TTableClient TableClient; - NYdb::NLongTx::TClient LongTxClient; NYdb::NTable::TSession Session; public: @@ -73,7 +71,6 @@ namespace NKqp { TTestActorRuntime& GetRuntime(); NYdb::NTable::TSession& GetSession(); void CreateTable(const TColumnTableBase& table); - void InsertData(const TColumnTable& table, TTestHelper::TUpdatesBuilder& updates, const std::function onBeforeCommit = {}, const NYdb::EStatus opStatus = NYdb::EStatus::SUCCESS); void BulkUpsert(const TColumnTable& table, TTestHelper::TUpdatesBuilder& updates, const Ydb::StatusIds_StatusCode& opStatus = Ydb::StatusIds::SUCCESS); void BulkUpsert(const TColumnTable& table, std::shared_ptr batch, const Ydb::StatusIds_StatusCode& opStatus = Ydb::StatusIds::SUCCESS); void ReadData(const TString& query, const TString& expected, const NYdb::EStatus opStatus = NYdb::EStatus::SUCCESS); diff --git a/ydb/core/kqp/ut/olap/kqp_olap_stats_ut.cpp b/ydb/core/kqp/ut/olap/kqp_olap_stats_ut.cpp index 2d34fb23496a..f77b5a019f20 100644 --- a/ydb/core/kqp/ut/olap/kqp_olap_stats_ut.cpp +++ b/ydb/core/kqp/ut/olap/kqp_olap_stats_ut.cpp @@ -48,7 +48,7 @@ Y_UNIT_TEST_SUITE(KqpOlapStats) { tableInserter.AddRow().Add(i).Add("test_res_" + std::to_string(i)).AddNull(); } - testHelper.InsertData(testTable, tableInserter); + testHelper.BulkUpsert(testTable, tableInserter); } Sleep(TDuration::Seconds(1)); @@ -88,7 +88,7 @@ Y_UNIT_TEST_SUITE(KqpOlapStats) { for (size_t i = 0; i < inserted_rows; i++) { tableInserter.AddRow().Add(i).Add("test_res_" + std::to_string(i)).AddNull(); } - testHelper.InsertData(testTable, tableInserter); + testHelper.BulkUpsert(testTable, tableInserter); } Sleep(TDuration::Seconds(1)); @@ -135,7 +135,7 @@ Y_UNIT_TEST_SUITE(KqpOlapStats) { .Add("test_res_" + std::to_string(i + t * tables_in_store)) .AddNull(); } - testHelper.InsertData(testTable, tableInserter); + testHelper.BulkUpsert(testTable, tableInserter); } Sleep(TDuration::Seconds(20)); @@ -155,4 +155,4 @@ Y_UNIT_TEST_SUITE(KqpOlapStats) { } } // namespace NKqp -} // namespace NKikimr \ No newline at end of file +} // namespace NKikimr diff --git a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp index a91b40588cf0..4013a0429d16 100644 --- a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp +++ b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include @@ -534,67 +533,25 @@ Y_UNIT_TEST_SUITE(KqpOlap) { void WriteTestData(TKikimrRunner& kikimr, TString testTable, ui64 pathIdBegin, ui64 tsBegin, size_t rowCount, bool withSomeNulls = false) { UNIT_ASSERT(testTable != "/Root/benchTable"); // TODO: check schema instead - TLocalHelper lHelper(kikimr); if (withSomeNulls) lHelper.WithSomeNulls(); - NYdb::NLongTx::TClient client(kikimr.GetDriver()); - - NLongTx::TLongTxBeginResult resBeginTx = client.BeginWriteTx().GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resBeginTx.Status().GetStatus(), EStatus::SUCCESS, resBeginTx.Status().GetIssues().ToString()); - - auto txId = resBeginTx.GetResult().tx_id(); auto batch = lHelper.TestArrowBatch(pathIdBegin, tsBegin, rowCount); - - TString data = NArrow::NSerialization::TFullDataSerializer(arrow::ipc::IpcWriteOptions::Defaults()).Serialize(batch); - - NLongTx::TLongTxWriteResult resWrite = - client.Write(txId, testTable, txId, data, Ydb::LongTx::Data::APACHE_ARROW).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resWrite.Status().GetStatus(), EStatus::SUCCESS, resWrite.Status().GetIssues().ToString()); - - NLongTx::TLongTxCommitResult resCommitTx = client.CommitTx(txId).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resCommitTx.Status().GetStatus(), EStatus::SUCCESS, resCommitTx.Status().GetIssues().ToString()); + lHelper.SendDataViaActorSystem(testTable, batch); } void WriteTestDataForClickBench(TKikimrRunner& kikimr, TString testTable, ui64 pathIdBegin, ui64 tsBegin, size_t rowCount) { UNIT_ASSERT(testTable == "/Root/benchTable"); // TODO: check schema instead - TClickHelper lHelper(kikimr.GetTestServer()); - NYdb::NLongTx::TClient client(kikimr.GetDriver()); - - NLongTx::TLongTxBeginResult resBeginTx = client.BeginWriteTx().GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resBeginTx.Status().GetStatus(), EStatus::SUCCESS, resBeginTx.Status().GetIssues().ToString()); - - auto txId = resBeginTx.GetResult().tx_id(); auto batch = lHelper.TestArrowBatch(pathIdBegin, tsBegin, rowCount); - TString data = NArrow::NSerialization::TFullDataSerializer(arrow::ipc::IpcWriteOptions::Defaults()).Serialize(batch); - - NLongTx::TLongTxWriteResult resWrite = - client.Write(txId, testTable, txId, data, Ydb::LongTx::Data::APACHE_ARROW).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resWrite.Status().GetStatus(), EStatus::SUCCESS, resWrite.Status().GetIssues().ToString()); - - NLongTx::TLongTxCommitResult resCommitTx = client.CommitTx(txId).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resCommitTx.Status().GetStatus(), EStatus::SUCCESS, resCommitTx.Status().GetIssues().ToString()); + lHelper.SendDataViaActorSystem(testTable, batch); } void WriteTestDataForTableWithNulls(TKikimrRunner& kikimr, TString testTable) { UNIT_ASSERT(testTable == "/Root/tableWithNulls"); // TODO: check schema instead TTableWithNullsHelper lHelper(kikimr.GetTestServer()); - NYdb::NLongTx::TClient client(kikimr.GetDriver()); - - NLongTx::TLongTxBeginResult resBeginTx = client.BeginWriteTx().GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resBeginTx.Status().GetStatus(), EStatus::SUCCESS, resBeginTx.Status().GetIssues().ToString()); - - auto txId = resBeginTx.GetResult().tx_id(); auto batch = lHelper.TestArrowBatch(); - TString data = NArrow::NSerialization::TFullDataSerializer(arrow::ipc::IpcWriteOptions::Defaults()).Serialize(batch); - - NLongTx::TLongTxWriteResult resWrite = - client.Write(txId, testTable, txId, data, Ydb::LongTx::Data::APACHE_ARROW).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resWrite.Status().GetStatus(), EStatus::SUCCESS, resWrite.Status().GetIssues().ToString()); - - NLongTx::TLongTxCommitResult resCommitTx = client.CommitTx(txId).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resCommitTx.Status().GetStatus(), EStatus::SUCCESS, resCommitTx.Status().GetIssues().ToString()); + lHelper.SendDataViaActorSystem(testTable, batch); } void CreateTableOfAllTypes(TKikimrRunner& kikimr) { @@ -5307,7 +5264,7 @@ Y_UNIT_TEST_SUITE(KqpOlap) { tableInserter.AddRow().Add(2).Add("test_res_2").Add("val1").AddNull(); tableInserter.AddRow().Add(3).Add("test_res_3").Add("val3").AddNull(); tableInserter.AddRow().Add(2).Add("test_res_2").Add("val2").AddNull(); - testHelper.InsertData(testTable, tableInserter); + testHelper.BulkUpsert(testTable, tableInserter); } while (csController->GetIndexations().Val() == 0) { Cout << "Wait indexation..." << Endl; diff --git a/ydb/core/kqp/ut/scheme/kqp_constraints_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_constraints_ut.cpp index 249a8a205d49..3e0cab5eb053 100644 --- a/ydb/core/kqp/ut/scheme/kqp_constraints_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_constraints_ut.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include diff --git a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp index 536595ccf922..bef64ef18078 100644 --- a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -5378,7 +5377,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema)); tableInserter.AddRow().Add(1).Add("test_res_1").AddNull(); tableInserter.AddRow().Add(2).Add("test_res_2").Add(123); - testHelper.InsertData(testTable, tableInserter); + testHelper.BulkUpsert(testTable, tableInserter); } testHelper.ReadData("SELECT * FROM `/Root/ColumnTableTest` WHERE id=1", "[[1;#;[\"test_res_1\"]]]"); @@ -5407,7 +5406,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { { TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema)); tableInserter.AddRow().Add(3).Add("test_res_3").Add(123).Add(200); - testHelper.InsertData(testTable, tableInserter); + testHelper.BulkUpsert(testTable, tableInserter); } testHelper.ReadData("SELECT * FROM `/Root/ColumnTableTest` WHERE id=3", "[[3;[123];[200u];[\"test_res_3\"]]]"); @@ -5444,7 +5443,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { } testHelper.ReadData("SELECT * FROM `/Root/ColumnTableTest` WHERE id=1", "[[1;#;#;[\"test_res_1\"]]]"); } - +/* Y_UNIT_TEST(AddColumnOnSchemeChange) { TKikimrSettings runnerSettings; runnerSettings.WithSampleTables = false; @@ -5471,7 +5470,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { } testHelper.ReadData("SELECT * FROM `/Root/ColumnTableTest` WHERE id=1", "[[1;#;#;[\"test_res_1\"]]]"); } - +*/ Y_UNIT_TEST(AddColumnWithStore) { TKikimrSettings runnerSettings; runnerSettings.WithSampleTables = false; @@ -5494,7 +5493,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema)); tableInserter.AddRow().Add(1).Add("test_res_1").AddNull(); tableInserter.AddRow().Add(2).Add("test_res_2").Add(123); - testHelper.InsertData(testTable, tableInserter); + testHelper.BulkUpsert(testTable, tableInserter); } testHelper.ReadData("SELECT * FROM `/Root/TableStoreTest/ColumnTableTest` WHERE id=1", "[[1;#;[\"test_res_1\"]]]"); @@ -5524,7 +5523,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { { TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema)); tableInserter.AddRow().Add(3).Add("test_res_3").Add(123).Add(200); - testHelper.InsertData(testTable, tableInserter); + testHelper.BulkUpsert(testTable, tableInserter); } testHelper.ReadData("SELECT * FROM `/Root/TableStoreTest/ColumnTableTest` WHERE id=3", "[[3;[123];[200u];[\"test_res_3\"]]]"); @@ -5580,7 +5579,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schemaWithNull)); tableInserter.AddRow().Add(1).Add("test_res_1").AddNull(); tableInserter.AddRow().Add(2).Add("test_res_2").Add(123); - testHelper.InsertData(testTable, tableInserter, {}, EStatus::GENERIC_ERROR); + testHelper.BulkUpsert(testTable, tableInserter, Ydb::StatusIds::GENERIC_ERROR); } { TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schemaWithNull)); @@ -5611,7 +5610,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema)); tableInserter.AddRow().Add(1).Add("test_res_1").AddNull(); tableInserter.AddRow().Add(2).Add("test_res_2").Add(123); - testHelper.InsertData(testTable, tableInserter); + testHelper.BulkUpsert(testTable, tableInserter); } testHelper.ReadData("SELECT * FROM `/Root/ColumnTableTest` WHERE id=1", "[[1;#;[\"test_res_1\"]]]"); { @@ -5628,7 +5627,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { } testHelper.ReadData("SELECT * FROM `/Root/ColumnTableTest` ", "[[1];[2]]"); } - +/* Y_UNIT_TEST(DropColumnOnSchemeChange) { TKikimrSettings runnerSettings; runnerSettings.WithSampleTables = false; @@ -5654,34 +5653,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { } testHelper.ReadData("SELECT * FROM `/Root/ColumnTableTest` WHERE id=1", "[[1;#]]"); } - - Y_UNIT_TEST(DropColumnOldScheme) { - TKikimrSettings runnerSettings; - runnerSettings.WithSampleTables = false; - TTestHelper testHelper(runnerSettings); - - TVector schema = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("resource_id").SetType(NScheme::NTypeIds::Utf8), - TTestHelper::TColumnSchema().SetName("level").SetType(NScheme::NTypeIds::Int32) - }; - - TTestHelper::TColumnTable testTable; - - testTable.SetName("/Root/ColumnTableTest").SetPrimaryKey({"id"}).SetSharding({"id"}).SetSchema(schema); - testHelper.CreateTable(testTable); - { - auto alterQuery = TStringBuilder() << "ALTER TABLE `" << testTable.GetName() << "`DROP COLUMN resource_id;"; - auto alterResult = testHelper.GetSession().ExecuteSchemeQuery(alterQuery).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), EStatus::SUCCESS, alterResult.GetIssues().ToString()); - } - { - TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema)); - tableInserter.AddRow().Add(1).Add("test_res_1").AddNull(); - testHelper.InsertData(testTable, tableInserter, {}, EStatus::SUCCESS); - } - // testHelper.ReadData("SELECT * FROM `/Root/ColumnTableTest` WHERE id=1", "[[1;#]]"); - } +*/ Y_UNIT_TEST(DropColumnOldSchemeBulkUpsert) { TKikimrSettings runnerSettings; @@ -5729,7 +5701,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema)); tableInserter.AddRow().Add(1).Add("test_res_1").AddNull(); tableInserter.AddRow().Add(2).Add("test_res_2").Add(123); - testHelper.InsertData(testTable, tableInserter); + testHelper.BulkUpsert(testTable, tableInserter); } testHelper.ReadData("SELECT * FROM `/Root/ColumnTableTest` WHERE id=1", "[[1;#;[\"test_res_1\"]]]"); { diff --git a/ydb/core/testlib/cs_helper.cpp b/ydb/core/testlib/cs_helper.cpp index 557d0cfff98f..4853290f8637 100644 --- a/ydb/core/testlib/cs_helper.cpp +++ b/ydb/core/testlib/cs_helper.cpp @@ -429,15 +429,14 @@ std::shared_ptr TTableWithNullsHelper::TestArrowBatch(ui64, Y_ABORT_UNLESS(bJsonDoc.AppendNull().ok()); } - auto maybeJsonDoc = NBinaryJson::SerializeToBinaryJson(R"({"col1": "val1", "obj": {"obj_col2_int": 16}})"); - Y_ABORT_UNLESS(maybeJsonDoc.Defined()); + const auto maybeJsonDoc = std::string(R"({"col1": "val1", "obj": {"obj_col2_int": 16}})"); for (size_t i = rowCount / 2 + 1; i <= rowCount; ++i) { Y_ABORT_UNLESS(bId.Append(i).ok()); Y_ABORT_UNLESS(bResourceId.Append(std::to_string(i)).ok()); Y_ABORT_UNLESS(bLevel.AppendNull().ok()); Y_ABORT_UNLESS(bBinaryStr.Append(std::to_string(i)).ok()); Y_ABORT_UNLESS(bJsonVal.AppendNull().ok()); - Y_ABORT_UNLESS(bJsonDoc.Append(maybeJsonDoc->Data(), maybeJsonDoc->Size()).ok()); + Y_ABORT_UNLESS(bJsonDoc.Append(maybeJsonDoc.data(), maybeJsonDoc.length()).ok()); } std::shared_ptr aId; diff --git a/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp b/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp index 08e4fad95c66..c5849c79f08c 100644 --- a/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp +++ b/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp @@ -222,7 +222,7 @@ TConclusionStatus TGeneralCompactColumnEngineChanges::DoConstructBlobs(TConstruc NChanges::TGeneralCompactionCounters::OnPortionsKind(insertedPortionsSize, compactedPortionsSize, otherPortionsSize); NChanges::TGeneralCompactionCounters::OnRepackPortions(portionsCount, portionsSize); - if (AppDataVerified().ColumnShardConfig.GetUseChunkedMergeOnCompaction()) { + if (!HasAppData() || AppDataVerified().ColumnShardConfig.GetUseChunkedMergeOnCompaction()) { BuildAppendedPortionsByChunks(context); } else { BuildAppendedPortionsByFullBatches(context); diff --git a/ydb/core/tx/columnshard/engines/ut_insert_table.cpp b/ydb/core/tx/columnshard/engines/ut_insert_table.cpp index 5f0848436ae5..ee10d7674b53 100644 --- a/ydb/core/tx/columnshard/engines/ut_insert_table.cpp +++ b/ydb/core/tx/columnshard/engines/ut_insert_table.cpp @@ -27,12 +27,12 @@ class TTestInsertTableDB : public IDbWrapper { return true; } - void WriteColumn(ui32, const TPortionInfo&, const TColumnRecord&) override {} - void EraseColumn(ui32, const TPortionInfo&, const TColumnRecord&) override {} - bool LoadColumns(ui32, const std::function&) override { return true; } + void WriteColumn(const TPortionInfo&, const TColumnRecord&) override {} + void EraseColumn(const TPortionInfo&, const TColumnRecord&) override {} + bool LoadColumns(const std::function&) override { return true; } - void WriteCounter(ui32, ui32, ui64) override {} - bool LoadCounters(ui32, const std::function&) override { return true; } + void WriteCounter(ui32, ui64) override {} + bool LoadCounters(const std::function&) override { return true; } }; } diff --git a/ydb/core/tx/columnshard/engines/ut_logs_engine.cpp b/ydb/core/tx/columnshard/engines/ut_logs_engine.cpp index 8e0c8ceef165..229100ecab5e 100644 --- a/ydb/core/tx/columnshard/engines/ut_logs_engine.cpp +++ b/ydb/core/tx/columnshard/engines/ut_logs_engine.cpp @@ -69,14 +69,14 @@ class TTestDbWrapper : public IDbWrapper { return true; } - void WriteColumn(ui32 index, const TPortionInfo& portion, const TColumnRecord& row) override { + void WriteColumn(const TPortionInfo& portion, const TColumnRecord& row) override { auto proto = portion.GetMeta().SerializeToProto(row.ColumnId, row.Chunk); auto rowProto = row.GetMeta().SerializeToProto(); if (proto) { *rowProto.MutablePortionMeta() = std::move(*proto); } - auto& data = Indices[index].Columns[portion.GetPathId()]; + auto& data = Indices[0].Columns[portion.GetPathId()]; NOlap::TColumnChunkLoadContext loadContext(row.GetAddress(), row.BlobRange, rowProto); auto itInsertInfo = LoadContexts[portion.GetAddress()].emplace(row.GetAddress(), loadContext); if (!itInsertInfo.second) { @@ -104,8 +104,8 @@ class TTestDbWrapper : public IDbWrapper { } } - void EraseColumn(ui32 index, const TPortionInfo& portion, const TColumnRecord& row) override { - auto& data = Indices[index].Columns[portion.GetPathId()]; + void EraseColumn(const TPortionInfo& portion, const TColumnRecord& row) override { + auto& data = Indices[0].Columns[portion.GetPathId()]; auto it = data.find(portion.GetPortion()); Y_ABORT_UNLESS(it != data.end()); auto& portionLocal = it->second; @@ -119,8 +119,8 @@ class TTestDbWrapper : public IDbWrapper { portionLocal.Records.swap(filtered); } - bool LoadColumns(ui32 index, const std::function& callback) override { - auto& columns = Indices[index].Columns; + bool LoadColumns(const std::function& callback) override { + auto& columns = Indices[0].Columns; for (auto& [pathId, portions] : columns) { for (auto& [portionId, portionLocal] : portions) { auto copy = portionLocal; @@ -137,13 +137,13 @@ class TTestDbWrapper : public IDbWrapper { return true; } - void WriteCounter(ui32 index, ui32 counterId, ui64 value) override { - auto& counters = Indices[index].Counters; + void WriteCounter(ui32 counterId, ui64 value) override { + auto& counters = Indices[0].Counters; counters[counterId] = value; } - bool LoadCounters(ui32 index, const std::function& callback) override { - auto& counters = Indices[index].Counters; + bool LoadCounters(const std::function& callback) override { + auto& counters = Indices[0].Counters; for (auto& [id, value] : counters) { callback(id, value); } diff --git a/ydb/core/tx/datashard/datashard_ut_trace.cpp b/ydb/core/tx/datashard/datashard_ut_trace.cpp index 50a5670865de..c8491d4ae513 100644 --- a/ydb/core/tx/datashard/datashard_ut_trace.cpp +++ b/ydb/core/tx/datashard/datashard_ut_trace.cpp @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include @@ -18,9 +18,9 @@ using namespace NKikimr::NDataShard::NKqpHelpers; using namespace NSchemeShard; using namespace Tests; using namespace NDataShardReadTableTest; +using namespace NWilson; Y_UNIT_TEST_SUITE(TDataShardTrace) { - void ExecSQL(Tests::TServer::TPtr server, TActorId sender, const TString &sql, @@ -108,154 +108,6 @@ Y_UNIT_TEST_SUITE(TDataShardTrace) { UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetRef().GetYdbStatus(), code); } - class FakeWilsonUploader : public TActorBootstrapped { - public: - class Span { - public: - Span(TString name, TString parentSpanId, ui64 startTime) : Name(name), ParentSpanId(parentSpanId), StartTime(startTime) {} - - std::optional> FindOne(TString targetName) { - for (const auto childRef : Children) { - if (childRef.get().Name == targetName) { - return childRef; - } - } - - return {}; - } - - std::vector> FindAll(TString targetName) { - std::vector> res; - - for (const auto childRef : Children) { - if (childRef.get().Name == targetName) { - res.emplace_back(childRef); - } - } - - return res; - } - - std::optional> BFSFindOne(TString targetName) { - std::queue> bfsQueue; - bfsQueue.push(std::ref(*this)); - - while (!bfsQueue.empty()) { - Span ¤tSpan = bfsQueue.front().get(); - bfsQueue.pop(); - - if (currentSpan.Name == targetName) { - return currentSpan; - } - - for (const auto childRef : currentSpan.Children) { - bfsQueue.push(childRef); - } - } - - return {}; - } - - static bool CompareByStartTime(const std::reference_wrapper& span1, const std::reference_wrapper& span2) { - return span1.get().StartTime < span2.get().StartTime; - } - - TString Name; - TString ParentSpanId; - ui64 StartTime; - std::set, decltype(&CompareByStartTime)> Children{&CompareByStartTime}; - }; - - class Trace { - public: - std::string ToString() const { - std::string result; - - for (const auto& spanPair : Spans) { - const Span& span = spanPair.second; - if (span.ParentSpanId.empty()) { - result += ToStringHelper(span); - } - } - - return result; - } - private: - std::string ToStringHelper(const Span& span) const { - std::string result = "(" + span.Name; - - if (!span.Children.empty()) { - result += " -> ["; - auto it = span.Children.begin(); - while (it != span.Children.end()) { - const Span& childSpan = it->get(); - result += ToStringHelper(childSpan); - ++it; - - if (it != span.Children.end()) { - result += " , "; - } - } - result += "]"; - } - - result += ")"; - - return result; - } - public: - std::unordered_map Spans; - - Span Root{"Root", "", 0}; - }; - - public: - void Bootstrap() { - Become(&TThis::StateFunc); - } - - void Handle(NWilson::TEvWilson::TPtr ev) { - auto& span = ev->Get()->Span; - const TString &traceId = span.trace_id(); - const TString &spanId = span.span_id(); - const TString &parentSpanId = span.parent_span_id(); - const TString &spanName = span.name(); - ui64 startTime = span.start_time_unix_nano(); - - Trace &trace = Traces[traceId]; - - trace.Spans.try_emplace(spanId, spanName, parentSpanId, startTime); - } - - void BuildTraceTrees() { - for (auto& tracePair : Traces) { - Trace& trace = tracePair.second; - - for (auto& spanPair : trace.Spans) { - Span& span = spanPair.second; - - const TString& parentSpanId = span.ParentSpanId; - - // Check if the span has a parent - if (!parentSpanId.empty()) { - auto parentSpanIt = trace.Spans.find(parentSpanId); - UNIT_ASSERT(parentSpanIt != trace.Spans.end()); - parentSpanIt->second.Children.insert(std::ref(span)); - } else { - trace.Root.Children.insert(std::ref(span)); - } - } - } - } - - STRICT_STFUNC(StateFunc, - hFunc(NWilson::TEvWilson, Handle); - ); - - public: - std::unordered_map Traces; - }; - void SplitTable(TTestActorRuntime &runtime, Tests::TServer::TPtr server, ui64 splitKey) { SetSplitMergePartCountLimit(server->GetRuntime(), -1); auto senderSplit = runtime.AllocateEdgeActor(); @@ -283,21 +135,21 @@ Y_UNIT_TEST_SUITE(TDataShardTrace) { return {runtime, server, sender}; } - void CheckTxHasWriteLog(std::reference_wrapper txSpan) { + void CheckTxHasWriteLog(std::reference_wrapper txSpan) { auto writeLogSpan = txSpan.get().FindOne("Tablet.WriteLog"); UNIT_ASSERT(writeLogSpan); auto writeLogEntrySpan = writeLogSpan->get().FindOne("Tablet.WriteLog.LogEntry"); UNIT_ASSERT(writeLogEntrySpan); } - void CheckTxHasDatashardUnits(std::reference_wrapper txSpan, ui8 count) { + void CheckTxHasDatashardUnits(std::reference_wrapper txSpan, ui8 count) { auto executeSpan = txSpan.get().FindOne("Tablet.Transaction.Execute"); UNIT_ASSERT(executeSpan); auto unitSpans = executeSpan->get().FindAll("Datashard.Unit"); UNIT_ASSERT_EQUAL(count, unitSpans.size()); } - void CheckExecuteHasDatashardUnits(std::reference_wrapper executeSpan, ui8 count) { + void CheckExecuteHasDatashardUnits(std::reference_wrapper executeSpan, ui8 count) { auto unitSpans = executeSpan.get().FindAll("Datashard.Unit"); UNIT_ASSERT_EQUAL(count, unitSpans.size()); } @@ -307,7 +159,7 @@ Y_UNIT_TEST_SUITE(TDataShardTrace) { CreateShardedTable(server, sender, "/Root", "table-1", 1, false); - FakeWilsonUploader *uploader = new FakeWilsonUploader(); + TFakeWilsonUploader *uploader = new TFakeWilsonUploader(); TActorId uploaderId = runtime.Register(uploader, 0); runtime.RegisterService(NWilson::MakeWilsonUploaderId(), uploaderId, 0); runtime.SimulateSleep(TDuration::Seconds(10)); @@ -323,11 +175,10 @@ Y_UNIT_TEST_SUITE(TDataShardTrace) { std::move(traceId) ); - uploader->BuildTraceTrees(); - + UNIT_ASSERT(uploader->BuildTraceTrees()); UNIT_ASSERT_EQUAL(1, uploader->Traces.size()); - FakeWilsonUploader::Trace &trace = uploader->Traces.begin()->second; + TFakeWilsonUploader::Trace &trace = uploader->Traces.begin()->second; auto deSpan = trace.Root.BFSFindOne("DataExecuter"); UNIT_ASSERT(deSpan); @@ -367,7 +218,7 @@ Y_UNIT_TEST_SUITE(TDataShardTrace) { CreateShardedTable(server, sender, "/Root", "table-1", 1, false); - FakeWilsonUploader *uploader = new FakeWilsonUploader(); + TFakeWilsonUploader *uploader = new TFakeWilsonUploader(); TActorId uploaderId = runtime.Register(uploader, 0); runtime.RegisterService(NWilson::MakeWilsonUploaderId(), uploaderId, 0); runtime.SimulateSleep(TDuration::Seconds(10)); @@ -414,11 +265,10 @@ Y_UNIT_TEST_SUITE(TDataShardTrace) { std::move(traceId) ); - uploader->BuildTraceTrees(); - + UNIT_ASSERT(uploader->BuildTraceTrees()); UNIT_ASSERT_EQUAL(1, uploader->Traces.size()); - FakeWilsonUploader::Trace &trace = uploader->Traces.begin()->second; + TFakeWilsonUploader::Trace &trace = uploader->Traces.begin()->second; std::string canon; if (server->GetSettings().AppConfig->GetTableServiceConfig().GetEnableKqpDataQueryStreamLookup()) { @@ -491,7 +341,7 @@ Y_UNIT_TEST_SUITE(TDataShardTrace) { CreateShardedTable(server, sender, "/Root", "table-1", 1, false); - FakeWilsonUploader* uploader = new FakeWilsonUploader(); + TFakeWilsonUploader* uploader = new TFakeWilsonUploader(); TActorId uploaderId = runtime.Register(uploader, 0); runtime.RegisterService(NWilson::MakeWilsonUploaderId(), uploaderId, 0); runtime.SimulateSleep(TDuration::Seconds(10)); @@ -522,11 +372,10 @@ Y_UNIT_TEST_SUITE(TDataShardTrace) { std::move(traceId) ); - uploader->BuildTraceTrees(); - + UNIT_ASSERT(uploader->BuildTraceTrees()); UNIT_ASSERT_EQUAL(1, uploader->Traces.size()); - FakeWilsonUploader::Trace& trace = uploader->Traces.begin()->second; + TFakeWilsonUploader::Trace& trace = uploader->Traces.begin()->second; auto readActorSpan = trace.Root.BFSFindOne("ReadActor"); UNIT_ASSERT(readActorSpan); @@ -551,7 +400,7 @@ Y_UNIT_TEST_SUITE(TDataShardTrace) { auto opts = TShardedTableOptions().Columns({{"key", "Uint32", true, false}, {"value", "Uint32", false, false}}); auto [shards, tableId] = CreateShardedTable(server, sender, "/Root", "table-1", opts); - FakeWilsonUploader *uploader = new FakeWilsonUploader(); + TFakeWilsonUploader *uploader = new TFakeWilsonUploader(); TActorId uploaderId = runtime.Register(uploader, 0); runtime.RegisterService(NWilson::MakeWilsonUploaderId(), uploaderId, 0); runtime.SimulateSleep(TDuration::Seconds(10)); @@ -561,11 +410,10 @@ Y_UNIT_TEST_SUITE(TDataShardTrace) { ui64 txId = 100; Write(runtime, sender, shards[0], tableId, opts.Columns_, rowCount, txId, NKikimrDataEvents::TEvWrite::MODE_IMMEDIATE, NKikimrDataEvents::TEvWriteResult::STATUS_UNSPECIFIED, std::move(traceId)); - uploader->BuildTraceTrees(); - + UNIT_ASSERT(uploader->BuildTraceTrees()); UNIT_ASSERT_EQUAL(1, uploader->Traces.size()); - FakeWilsonUploader::Trace &trace = uploader->Traces.begin()->second; + TFakeWilsonUploader::Trace &trace = uploader->Traces.begin()->second; auto wtSpan = trace.Root.BFSFindOne("Datashard.WriteTransaction"); UNIT_ASSERT(wtSpan); diff --git a/ydb/library/actors/wilson/test_util/fake_wilson_uploader.h b/ydb/library/actors/wilson/test_util/fake_wilson_uploader.h new file mode 100644 index 000000000000..9afc2a2e68dc --- /dev/null +++ b/ydb/library/actors/wilson/test_util/fake_wilson_uploader.h @@ -0,0 +1,164 @@ +#pragma once + +#include +#include +#include + +namespace NWilson { + + class TFakeWilsonUploader : public NActors::TActorBootstrapped { + public: + class Span { + public: + Span(TString name, TString parentSpanId, ui64 startTime) : Name(name), ParentSpanId(parentSpanId), StartTime(startTime) {} + + std::optional> FindOne(TString targetName) { + for (const auto childRef : Children) { + if (childRef.get().Name == targetName) { + return childRef; + } + } + + return {}; + } + + std::vector> FindAll(TString targetName) { + std::vector> res; + + for (const auto childRef : Children) { + if (childRef.get().Name == targetName) { + res.emplace_back(childRef); + } + } + + return res; + } + + std::optional> BFSFindOne(TString targetName) { + std::queue> bfsQueue; + bfsQueue.push(std::ref(*this)); + + while (!bfsQueue.empty()) { + Span ¤tSpan = bfsQueue.front().get(); + bfsQueue.pop(); + + if (currentSpan.Name == targetName) { + return currentSpan; + } + + for (const auto childRef : currentSpan.Children) { + bfsQueue.push(childRef); + } + } + + return {}; + } + + static bool CompareByStartTime(const std::reference_wrapper& span1, const std::reference_wrapper& span2) { + return span1.get().StartTime < span2.get().StartTime; + } + + TString Name; + TString ParentSpanId; + ui64 StartTime; + std::set, decltype(&CompareByStartTime)> Children{&CompareByStartTime}; + }; + + class Trace { + public: + std::string ToString() const { + std::string result; + + for (const auto& spanPair : Spans) { + const Span& span = spanPair.second; + if (span.ParentSpanId.empty()) { + result += ToStringHelper(span); + } + } + + return result; + } + private: + std::string ToStringHelper(const Span& span) const { + std::string result = "(" + span.Name; + + if (!span.Children.empty()) { + result += " -> ["; + auto it = span.Children.begin(); + while (it != span.Children.end()) { + const Span& childSpan = it->get(); + result += ToStringHelper(childSpan); + ++it; + + if (it != span.Children.end()) { + result += " , "; + } + } + result += "]"; + } + + result += ")"; + + return result; + } + public: + std::unordered_map Spans; + + Span Root{"Root", "", 0}; + }; + + public: + void Bootstrap() { + Become(&TThis::StateFunc); + } + + void Handle(NWilson::TEvWilson::TPtr ev) { + auto& span = ev->Get()->Span; + const TString &traceId = span.trace_id(); + const TString &spanId = span.span_id(); + const TString &parentSpanId = span.parent_span_id(); + const TString &spanName = span.name(); + ui64 startTime = span.start_time_unix_nano(); + + Trace &trace = Traces[traceId]; + + trace.Spans.try_emplace(spanId, spanName, parentSpanId, startTime); + } + + [[nodiscard]] bool BuildTraceTrees() { + for (auto& tracePair : Traces) { + Trace& trace = tracePair.second; + + for (auto& spanPair : trace.Spans) { + Span& span = spanPair.second; + + const TString& parentSpanId = span.ParentSpanId; + + // Check if the span has a parent + if (!parentSpanId.empty()) { + auto parentSpanIt = trace.Spans.find(parentSpanId); + if (parentSpanIt == trace.Spans.end()) { + return false; + } + parentSpanIt->second.Children.insert(std::ref(span)); + } else { + trace.Root.Children.insert(std::ref(span)); + } + } + } + return true; + } + + void Clear() { + Traces.clear(); + } + + STRICT_STFUNC(StateFunc, + hFunc(NWilson::TEvWilson, Handle); + ); + + public: + std::unordered_map Traces; + }; + +} // NWilson diff --git a/ydb/library/actors/wilson/test_util/ya.make b/ydb/library/actors/wilson/test_util/ya.make new file mode 100644 index 000000000000..4ff8f8c26e86 --- /dev/null +++ b/ydb/library/actors/wilson/test_util/ya.make @@ -0,0 +1,7 @@ +LIBRARY() + +SRCS( + fake_wilson_uploader.h +) + +END() diff --git a/ydb/library/actors/wilson/ya.make b/ydb/library/actors/wilson/ya.make index 9786754fae78..07524c01b151 100644 --- a/ydb/library/actors/wilson/ya.make +++ b/ydb/library/actors/wilson/ya.make @@ -22,4 +22,5 @@ RECURSE( RECURSE_FOR_TESTS( ut + test_util ) diff --git a/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp b/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp index 84f03a737d81..62ff4aa5c1a6 100644 --- a/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp +++ b/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp @@ -462,6 +462,14 @@ class TReturnSetInfo { } ~TReturnSetInfo() { + Free(); + } + + void Free() { + if (!Ptr) { + return; + } + if (Ref().expectedDesc) { FreeTupleDesc(Ref().expectedDesc); } @@ -471,9 +479,12 @@ class TReturnSetInfo { } TWithDefaultMiniKQLAlloc::FreeWithSize(Ptr, sizeof(ReturnSetInfo)); + Ptr = nullptr; } ReturnSetInfo& Ref() { + Y_ENSURE(Ptr, "ReturnSetInfo is dead"); + return *static_cast(Ptr); } @@ -488,11 +499,21 @@ class TExprContextHolder { } ExprContext& Ref() { + Y_ENSURE(Ptr, "TExprContextHolder is dead"); + return *Ptr; } ~TExprContextHolder() { + Free(); + } + + void Free() { + if (!Ptr) { + return; + } FreeExprContext(Ptr, true); + Ptr = nullptr; } private: @@ -739,9 +760,7 @@ class TPgResolvedMultiCall : public TPgResolvedCallBase { } ~TIterator() { - if (TupleSlot) { - ExecDropSingleTupleTableSlot(TupleSlot); - } + FinishAndFree(); } private: @@ -766,7 +785,7 @@ class TPgResolvedMultiCall : public TPgResolvedCallBase { } else { YQL_ENSURE(!StructType); if (RSInfo.Ref().isDone == ExprEndResult) { - IsFinished = true; + FinishAndFree(); return false; } @@ -782,7 +801,7 @@ class TPgResolvedMultiCall : public TPgResolvedCallBase { bool CopyTuple(NUdf::TUnboxedValue& value) { if (!tuplestore_gettupleslot(RSInfo.Ref().setResult, true, false, TupleSlot)) { - IsFinished = true; + FinishAndFree(); return false; } @@ -835,6 +854,17 @@ class TPgResolvedMultiCall : public TPgResolvedCallBase { } } + void FinishAndFree() { + if (TupleSlot) { + ExecDropSingleTupleTableSlot(TupleSlot); + TupleSlot = nullptr; + } + RSInfo.Free(); + ExprContextHolder.Free(); + + IsFinished = true; + } + const std::string_view Name; TUnboxedValueVector Args; const TVector& ArgDesc; diff --git a/ydb/library/yql/tests/postgresql/cases/aggregates.err b/ydb/library/yql/tests/postgresql/cases/aggregates.err index 695de54f18e3..6668a65fb068 100644 --- a/ydb/library/yql/tests/postgresql/cases/aggregates.err +++ b/ydb/library/yql/tests/postgresql/cases/aggregates.err @@ -654,12 +654,7 @@ COPY bitwise_test FROM STDIN NULL 'null'; 7 7 7 3 4 B1100 \. --stdin-:
: Error: Parse Sql - - -stdin-:
:2:1: Error: ERROR: syntax error at or near "1" - - 1 1 1 1 1 B0101 - ^ +Metacommand \. is not supported SELECT BIT_AND(i2) AS "1", @@ -769,12 +764,7 @@ FALSE TRUE null null null TRUE FALSE null \. --stdin-:
: Error: Parse Sql - - -stdin-:
:2:1: Error: ERROR: syntax error at or near "TRUE" - - TRUE null FALSE null - ^ +Metacommand \. is not supported SELECT BOOL_AND(b1) AS "f", @@ -1354,4 +1344,4 @@ drop table p_t1; -- create temp table t1(f1 int, f2 bigint); -(TFileError) (File exists) util/system/file.cpp:857: can't open "/tmp/tmpbbmu_5di/t1" with mode CreateNew (0x00000003) +(TFileError) (File exists) util/system/file.cpp:918: can't open "/tmp/tmpjl487dvf/t1" with mode CreateNew (0x00000003) diff --git a/ydb/library/yql/tests/postgresql/cases/aggregates.out b/ydb/library/yql/tests/postgresql/cases/aggregates.out index 6bf57a88141a..be38f5ac4985 100644 --- a/ydb/library/yql/tests/postgresql/cases/aggregates.out +++ b/ydb/library/yql/tests/postgresql/cases/aggregates.out @@ -53,6 +53,67 @@ SELECT stddev_pop('nan'::numeric), stddev_samp('nan'::numeric); NaN | (1 row) +-- verify correct results for null and NaN inputs +select sum(null::int4) from generate_series(1,3); + sum +----- + +(1 row) + +select sum(null::int8) from generate_series(1,3); + sum +----- + +(1 row) + +select sum(null::numeric) from generate_series(1,3); + sum +----- + +(1 row) + +select sum(null::float8) from generate_series(1,3); + sum +----- + +(1 row) + +select avg(null::int4) from generate_series(1,3); + avg +----- + +(1 row) + +select avg(null::int8) from generate_series(1,3); + avg +----- + +(1 row) + +select avg(null::numeric) from generate_series(1,3); + avg +----- + +(1 row) + +select avg(null::float8) from generate_series(1,3); + avg +----- + +(1 row) + +select sum('NaN'::numeric) from generate_series(1,3); + sum +----- + NaN +(1 row) + +select avg('NaN'::numeric) from generate_series(1,3); + avg +----- + NaN +(1 row) + SELECT sum(x::numeric), avg(x::numeric), var_pop(x::numeric) FROM (VALUES ('1'), ('infinity')) v(x); sum | avg | var_pop diff --git a/ydb/library/yql/tests/postgresql/cases/aggregates.sql b/ydb/library/yql/tests/postgresql/cases/aggregates.sql index b1c7aa24889e..10ef58550ba8 100644 --- a/ydb/library/yql/tests/postgresql/cases/aggregates.sql +++ b/ydb/library/yql/tests/postgresql/cases/aggregates.sql @@ -13,6 +13,17 @@ SELECT var_pop('inf'::numeric), var_samp('inf'::numeric); SELECT stddev_pop('inf'::numeric), stddev_samp('inf'::numeric); SELECT var_pop('nan'::numeric), var_samp('nan'::numeric); SELECT stddev_pop('nan'::numeric), stddev_samp('nan'::numeric); +-- verify correct results for null and NaN inputs +select sum(null::int4) from generate_series(1,3); +select sum(null::int8) from generate_series(1,3); +select sum(null::numeric) from generate_series(1,3); +select sum(null::float8) from generate_series(1,3); +select avg(null::int4) from generate_series(1,3); +select avg(null::int8) from generate_series(1,3); +select avg(null::numeric) from generate_series(1,3); +select avg(null::float8) from generate_series(1,3); +select sum('NaN'::numeric) from generate_series(1,3); +select avg('NaN'::numeric) from generate_series(1,3); SELECT sum(x::numeric), avg(x::numeric), var_pop(x::numeric) FROM (VALUES ('1'), ('infinity')) v(x); SELECT sum(x::numeric), avg(x::numeric), var_pop(x::numeric) diff --git a/ydb/library/yql/tests/postgresql/cases/bit.err b/ydb/library/yql/tests/postgresql/cases/bit.err index 836e062b35a9..1ed31207ff8c 100644 --- a/ydb/library/yql/tests/postgresql/cases/bit.err +++ b/ydb/library/yql/tests/postgresql/cases/bit.err @@ -138,12 +138,7 @@ XFA50 X05AF X1234 XFFF5 \. --stdin-:
: Error: Parse Sql - - -stdin-:
:2:1: Error: ERROR: syntax error at or near "X0F" - - X0F X10 - ^ +Metacommand \. is not supported SELECT a, b, ~a AS "~ a", a & b AS "a & b", a | b AS "a | b", a # b AS "a # b" FROM varbit_table; @@ -179,12 +174,7 @@ XFA50 X05AF X1234 XFFF5 \. --stdin-:
: Error: Parse Sql - - -stdin-:
:2:1: Error: ERROR: syntax error at or near "X0F00" - - X0F00 X1000 - ^ +Metacommand \. is not supported SELECT a,b,~a AS "~ a",a & b AS "a & b", a|b AS "a | b", a # b AS "a # b" FROM bit_table; diff --git a/ydb/library/yql/tests/postgresql/cases/create_table.err b/ydb/library/yql/tests/postgresql/cases/create_table.err index bfc9390ba166..74172d95df30 100644 --- a/ydb/library/yql/tests/postgresql/cases/create_table.err +++ b/ydb/library/yql/tests/postgresql/cases/create_table.err @@ -434,23 +434,13 @@ DEALLOCATE select1; -- (temporarily hide query, to avoid the long CREATE TABLE stmt) \set ECHO none --stdin-:
: Error: Parse Sql - - -stdin-:
:3:1: Error: ERROR: syntax error at or near "\" - - \set ECHO none - ^ +Metacommand \set ECHO none is not supported SELECT 'CREATE TABLE extra_wide_table(firstc text, '|| array_to_string(array_agg('c'||i||' bool'),',')||', lastc text);' FROM generate_series(1, 1100) g(i) \gexec --stdin-:
: Error: Parse Sql - - -stdin-:
:3:1: Error: ERROR: syntax error at or near "\" - - \gexec - ^ +Metacommand \gexec is not supported \set ECHO all @@ -978,12 +968,7 @@ CREATE TABLE fail () INHERITS (partitioned2); -- Partition key in describe output \d partitioned --stdin-:
: Error: Parse Sql - - -stdin-:
:2:1: Error: ERROR: syntax error at or near "\" - - \d partitioned - ^ +Metacommand \d partitioned is not supported \d+ partitioned2 @@ -2274,12 +2259,7 @@ insert into parted_notnull_inh_test (b) values (null); -- note that while b's default is overriden, a's default is preserved \d parted_notnull_inh_test1 --stdin-:
: Error: Parse Sql - - -stdin-:
:2:1: Error: ERROR: syntax error at or near "\" - - \d parted_notnull_inh_test1 - ^ +Metacommand \d parted_notnull_inh_test1 is not supported drop table parted_notnull_inh_test; @@ -2399,32 +2379,17 @@ drop table test_part_coll_posix; -- Partition bound in describe output \d+ part_b --stdin-:
: Error: Parse Sql - - -stdin-:
:2:1: Error: ERROR: syntax error at or near "\" - - \d+ part_b - ^ +Metacommand \d+ part_b is not supported -- Both partition bound and partition key in describe output \d+ part_c --stdin-:
: Error: Parse Sql - - -stdin-:
:2:1: Error: ERROR: syntax error at or near "\" - - \d+ part_c - ^ +Metacommand \d+ part_c is not supported -- a level-2 partition's constraint will include the parent's expressions \d+ part_c_1_10 --stdin-:
: Error: Parse Sql - - -stdin-:
:2:1: Error: ERROR: syntax error at or near "\" - - \d+ part_c_1_10 - ^ +Metacommand \d+ part_c_1_10 is not supported -- Show partition count in the parent's describe output -- Tempted to include \d+ output listing partitions with bound info but @@ -2432,12 +2397,7 @@ drop table test_part_coll_posix; -- returned. \d parted --stdin-:
: Error: Parse Sql - - -stdin-:
:5:1: Error: ERROR: syntax error at or near "\" - - \d parted - ^ +Metacommand \d parted is not supported \d hash_parted diff --git a/ydb/library/yql/tests/postgresql/cases/functional_deps.err b/ydb/library/yql/tests/postgresql/cases/functional_deps.err index f375d08dea5b..fc6bed0daed9 100644 --- a/ydb/library/yql/tests/postgresql/cases/functional_deps.err +++ b/ydb/library/yql/tests/postgresql/cases/functional_deps.err @@ -455,14 +455,6 @@ INNER JOIN users u ON u.uid = n.uid WHERE n.type = 'blog' AND n.status = 1 GROUP BY u.uid, u.name; --stdin-:
: Error: Type annotation - - -stdin-:
:1:1: Error: At function: AssumeColumnOrder, At function: PgReplaceUnknown, At function: OrderedMap, At function: UnionAll, At function: Aggregate - -- OK - ^ - -stdin-:
:1:1: Error: Expected hashable and equatable type for key column: _alias_u.name, but got: pgvarchar - -- OK - ^ -- OK SELECT u.uid, u.name FROM node n diff --git a/ydb/library/yql/tests/postgresql/cases/functional_deps.out b/ydb/library/yql/tests/postgresql/cases/functional_deps.out index 3f1fb1b9b052..60ac758a4294 100644 --- a/ydb/library/yql/tests/postgresql/cases/functional_deps.out +++ b/ydb/library/yql/tests/postgresql/cases/functional_deps.out @@ -35,3 +35,12 @@ CREATE TEMP TABLE users ( PRIMARY KEY (uid), UNIQUE (name) ); +-- OK +SELECT u.uid, u.name FROM node n +INNER JOIN users u ON u.uid = n.uid +WHERE n.type = 'blog' AND n.status = 1 +GROUP BY u.uid, u.name; + uid | name +-----+------ +(0 rows) + diff --git a/ydb/library/yql/tests/postgresql/cases/functional_deps.sql b/ydb/library/yql/tests/postgresql/cases/functional_deps.sql index 3f1fb1b9b052..4c8da83ef895 100644 --- a/ydb/library/yql/tests/postgresql/cases/functional_deps.sql +++ b/ydb/library/yql/tests/postgresql/cases/functional_deps.sql @@ -35,3 +35,8 @@ CREATE TEMP TABLE users ( PRIMARY KEY (uid), UNIQUE (name) ); +-- OK +SELECT u.uid, u.name FROM node n +INNER JOIN users u ON u.uid = n.uid +WHERE n.type = 'blog' AND n.status = 1 +GROUP BY u.uid, u.name; diff --git a/ydb/library/yql/tests/postgresql/cases/insert.err b/ydb/library/yql/tests/postgresql/cases/insert.err index f1a338b139c9..9939f85884bc 100644 --- a/ydb/library/yql/tests/postgresql/cases/insert.err +++ b/ydb/library/yql/tests/postgresql/cases/insert.err @@ -61,23 +61,15 @@ insert into inserttest (col1, col2, col3) values (DEFAULT, DEFAULT); insert into inserttest (col1, col2, col3) values (1, 2); --stdin-:
: Error: Type annotation - - -stdin-:
:1:1: Error: At function: RemovePrefixMembers, At function: PgSelect, At function: PgSetItem - insert into inserttest (col1, col2, col3) values (1, 2); - ^ - -stdin-:
:1:1: Error: values and target_columns sizes do not match - insert into inserttest (col1, col2, col3) values (1, 2); - ^ insert into inserttest (col1) values (1, 2); -stdin-:
: Error: Type annotation - -stdin-:
:1:1: Error: At function: RemovePrefixMembers, At function: PgSelect, At function: PgSetItem + -stdin-:
:1:1: Error: At function: YtWriteTable! insert into inserttest (col1) values (1, 2); ^ - -stdin-:
:1:1: Error: values and target_columns sizes do not match + -stdin-:
:1:1: Error: values have 2 columns, INSERT INTO expects: 1 insert into inserttest (col1) values (1, 2); ^ @@ -1185,12 +1177,7 @@ from hash_parted order by part; -- partitions \d+ list_parted --stdin-:
: Error: Parse Sql - - -stdin-:
:3:1: Error: ERROR: syntax error at or near "\" - - \d+ list_parted - ^ +Metacommand \d+ list_parted is not supported -- cleanup drop table range_parted, list_parted; @@ -2505,12 +2492,7 @@ copy donothingbrtrig_test from stdout; 2 qux \. --stdin-:
: Error: Parse Sql - - -stdin-:
:2:1: Error: ERROR: syntax error at or near "1" - - 1 baz - ^ +Metacommand \. is not supported select tableoid::regclass, * from donothingbrtrig_test; diff --git a/ydb/library/yql/tests/postgresql/cases/int8.err b/ydb/library/yql/tests/postgresql/cases/int8.err index 5b962d292809..82decd997cc1 100644 --- a/ydb/library/yql/tests/postgresql/cases/int8.err +++ b/ydb/library/yql/tests/postgresql/cases/int8.err @@ -717,6 +717,6 @@ SELECT * FROM generate_series('+4567890123456789'::int8, '+4567890123456799'::in SELECT * FROM generate_series('+4567890123456789'::int8, '+4567890123456799'::int8, 2); -pgrun: /storage/d/arcc/mount/1/a0a05244-a249bf61-ec2999f5-6661a47b/contrib/libs/llvm12/lib/IR/LegacyPassManager.cpp:588: void llvm::PMTopLevelManager::setLastUser(ArrayRef, Pass *): Assertion `AnalysisPass && "Expected analysis pass to exist."' failed. +pgrun: /-S/ydbwork/ydb/contrib/libs/llvm12/lib/IR/LegacyPassManager.cpp:588: void llvm::PMTopLevelManager::setLastUser(ArrayRef, Pass *): Assertion `AnalysisPass && "Expected analysis pass to exist."' failed. pthread_kill at ./nptl/./nptl/pthread_kill.c:43:17 ?? at ??:0:0 diff --git a/ydb/library/yql/tests/postgresql/cases/interval.err b/ydb/library/yql/tests/postgresql/cases/interval.err index 2bf468218362..b67f8dd6a134 100644 --- a/ydb/library/yql/tests/postgresql/cases/interval.err +++ b/ydb/library/yql/tests/postgresql/cases/interval.err @@ -286,12 +286,7 @@ COPY INTERVAL_MULDIV_TBL FROM STDIN; 999 mon 999 days \. --stdin-:
: Error: Parse Sql - - -stdin-:
:2:1: Error: ERROR: syntax error at or near "41" - - 41 mon 12 days 360:00 - ^ +Metacommand \. is not supported SELECT span * 0.3 AS product FROM INTERVAL_MULDIV_TBL; diff --git a/ydb/library/yql/tests/postgresql/cases/join.err b/ydb/library/yql/tests/postgresql/cases/join.err index 50d0e21eb6b7..e468af68aea9 100644 --- a/ydb/library/yql/tests/postgresql/cases/join.err +++ b/ydb/library/yql/tests/postgresql/cases/join.err @@ -1680,4 +1680,4 @@ create temp table a ( constraint a_pk primary key (code) ); -(TFileError) (File exists) util/system/file.cpp:857: can't open "/tmp/tmpglg8to66/a" with mode CreateNew (0x00000003) +(TFileError) (File exists) util/system/file.cpp:918: can't open "/tmp/tmp3znkmfbo/a" with mode CreateNew (0x00000003) diff --git a/ydb/library/yql/tests/postgresql/cases/jsonb.err b/ydb/library/yql/tests/postgresql/cases/jsonb.err index d2cbf6610b70..c0a44f50465a 100644 --- a/ydb/library/yql/tests/postgresql/cases/jsonb.err +++ b/ydb/library/yql/tests/postgresql/cases/jsonb.err @@ -5938,12 +5938,6 @@ select jsonb_set('{"a": {"b": [1, 2, 3]}}', '{a, b, NULL}', '"new_value"'); -- jsonb_set_lax \pset null NULL --stdin-:
: Error: Parse Sql - - -stdin-:
:2:1: Error: ERROR: syntax error at or near "\" - - \pset null NULL - ^ -- pass though non nulls to jsonb_set select jsonb_set_lax('{"a":1,"b":2}','{b}','5') ; @@ -6063,7 +6057,6 @@ select jsonb_set_lax('{"a":1,"b":2}', '{b}', null, null_value_treatment => 'use_ \pset null '' -Metacommand \pset null '' is not supported -- jsonb_insert select jsonb_insert('{"a": [0,1,2]}', '{a, 1}', '"new_value"'); diff --git a/ydb/library/yql/tests/postgresql/cases/numeric.err b/ydb/library/yql/tests/postgresql/cases/numeric.err index 13bbc7245481..421fe55e56e1 100644 --- a/ydb/library/yql/tests/postgresql/cases/numeric.err +++ b/ydb/library/yql/tests/postgresql/cases/numeric.err @@ -2772,12 +2772,7 @@ COPY width_bucket_test (operand_num) FROM stdin; 10.0000000000001 \. --stdin-:
: Error: Parse Sql - - -stdin-:
:2:1: Error: ERROR: syntax error at or near "-" - - -5.2 - ^ +Metacommand \. is not supported UPDATE width_bucket_test SET operand_f8 = operand_num::float8; @@ -3617,6 +3612,6 @@ select * from generate_series(-100::numeric, 100::numeric, 0::numeric); select * from generate_series(-100::numeric, 100::numeric, 'nan'::numeric); -pgrun: /storage/d/arcc/mount/1/a0a05244-a249bf61-ec2999f5-6661a47b/contrib/libs/llvm12/lib/IR/LegacyPassManager.cpp:588: void llvm::PMTopLevelManager::setLastUser(ArrayRef, Pass *): Assertion `AnalysisPass && "Expected analysis pass to exist."' failed. +pgrun: /-S/ydbwork/ydb/contrib/libs/llvm12/lib/IR/LegacyPassManager.cpp:588: void llvm::PMTopLevelManager::setLastUser(ArrayRef, Pass *): Assertion `AnalysisPass && "Expected analysis pass to exist."' failed. pthread_kill at ./nptl/./nptl/pthread_kill.c:43:17 ?? at ??:0:0 diff --git a/ydb/library/yql/tests/postgresql/cases/subselect.err b/ydb/library/yql/tests/postgresql/cases/subselect.err index 6dfa1746d69d..566d00bef634 100644 --- a/ydb/library/yql/tests/postgresql/cases/subselect.err +++ b/ydb/library/yql/tests/postgresql/cases/subselect.err @@ -4,36 +4,12 @@ -- SELECT 1 AS one WHERE 1 IN (SELECT 1); --stdin-:
: Fatal: Optimization - - -stdin-:
:4:25: Fatal: ydb/library/yql/core/common_opt/yql_co_pgselect.cpp:2852 JoinOuter(): requirement outerInputs.size() == finalExtTypes->Tail().ChildrenSize() failed - SELECT 1 AS one WHERE 1 IN (SELECT 1); - ^ - -stdin-:
:1:1: Fatal: ydb/library/yql/core/common_opt/yql_co_pgselect.cpp:640 RewriteSubLinksPartial(): requirement status.Level != IGraphTransformer::TStatus::Error failed - -- - ^ SELECT 1 AS zero WHERE 1 NOT IN (SELECT 1); --stdin-:
: Fatal: Optimization - - -stdin-:
:1:26: Fatal: ydb/library/yql/core/common_opt/yql_co_pgselect.cpp:2852 JoinOuter(): requirement outerInputs.size() == finalExtTypes->Tail().ChildrenSize() failed - SELECT 1 AS zero WHERE 1 NOT IN (SELECT 1); - ^ - -stdin-:
:1:1: Fatal: ydb/library/yql/core/common_opt/yql_co_pgselect.cpp:640 RewriteSubLinksPartial(): requirement status.Level != IGraphTransformer::TStatus::Error failed - SELECT 1 AS zero WHERE 1 NOT IN (SELECT 1); - ^ SELECT 1 AS zero WHERE 1 IN (SELECT 2); --stdin-:
: Fatal: Optimization - - -stdin-:
:1:26: Fatal: ydb/library/yql/core/common_opt/yql_co_pgselect.cpp:2852 JoinOuter(): requirement outerInputs.size() == finalExtTypes->Tail().ChildrenSize() failed - SELECT 1 AS zero WHERE 1 IN (SELECT 2); - ^ - -stdin-:
:1:1: Fatal: ydb/library/yql/core/common_opt/yql_co_pgselect.cpp:640 RewriteSubLinksPartial(): requirement status.Level != IGraphTransformer::TStatus::Error failed - SELECT 1 AS zero WHERE 1 IN (SELECT 2); - ^ -- Check grammar's handling of extra parens in assorted contexts SELECT * FROM (SELECT 1 AS x) ss; @@ -44,5 +20,5 @@ SELECT * FROM ((SELECT 1 AS x)) ss; (SELECT 2) UNION SELECT 2; -NYql::TAggregateExpander::ExpandAggApply(TIntrusivePtr > const&) at /-S/ydb/library/yql/core/yql_aggregate_expander.cpp:119:51 +NYql::TAggregateExpander::ExpandAggApply(TIntrusivePtr > const&) at /-S/ydbwork/ydb/ydb/library/yql/core/yql_aggregate_expander.cpp:119:51 ?? at ??:0:0 diff --git a/ydb/library/yql/tests/postgresql/cases/subselect.out b/ydb/library/yql/tests/postgresql/cases/subselect.out index e5202b1d2067..9fbeb65df2d7 100644 --- a/ydb/library/yql/tests/postgresql/cases/subselect.out +++ b/ydb/library/yql/tests/postgresql/cases/subselect.out @@ -1,3 +1,22 @@ +-- +-- SUBSELECT +-- +SELECT 1 AS one WHERE 1 IN (SELECT 1); + one +----- + 1 +(1 row) + +SELECT 1 AS zero WHERE 1 NOT IN (SELECT 1); + zero +------ +(0 rows) + +SELECT 1 AS zero WHERE 1 IN (SELECT 2); + zero +------ +(0 rows) + -- Check grammar's handling of extra parens in assorted contexts SELECT * FROM (SELECT 1 AS x) ss; x diff --git a/ydb/library/yql/tests/postgresql/cases/subselect.sql b/ydb/library/yql/tests/postgresql/cases/subselect.sql index 95985320955e..cc4abd17a6a3 100644 --- a/ydb/library/yql/tests/postgresql/cases/subselect.sql +++ b/ydb/library/yql/tests/postgresql/cases/subselect.sql @@ -1,3 +1,9 @@ +-- +-- SUBSELECT +-- +SELECT 1 AS one WHERE 1 IN (SELECT 1); +SELECT 1 AS zero WHERE 1 NOT IN (SELECT 1); +SELECT 1 AS zero WHERE 1 IN (SELECT 2); -- Check grammar's handling of extra parens in assorted contexts SELECT * FROM (SELECT 1 AS x) ss; SELECT * FROM ((SELECT 1 AS x)) ss; diff --git a/ydb/library/yql/tests/postgresql/cases/timestamptz.err b/ydb/library/yql/tests/postgresql/cases/timestamptz.err index 7ba6e87295a1..263087a9d0d7 100644 --- a/ydb/library/yql/tests/postgresql/cases/timestamptz.err +++ b/ydb/library/yql/tests/postgresql/cases/timestamptz.err @@ -1142,6 +1142,6 @@ SET TimeZone to 'UTC'; SELECT '2011-03-27 00:00:00 Europe/Moscow'::timestamptz; -pgrun: /storage/d/arcc/mount/1/a0a05244-a249bf61-ec2999f5-6661a47b/contrib/libs/llvm12/lib/IR/LegacyPassManager.cpp:588: void llvm::PMTopLevelManager::setLastUser(ArrayRef, Pass *): Assertion `AnalysisPass && "Expected analysis pass to exist."' failed. +pgrun: /-S/ydbwork/ydb/contrib/libs/llvm12/lib/IR/LegacyPassManager.cpp:588: void llvm::PMTopLevelManager::setLastUser(ArrayRef, Pass *): Assertion `AnalysisPass && "Expected analysis pass to exist."' failed. pthread_kill at ./nptl/./nptl/pthread_kill.c:43:17 ?? at ??:0:0 diff --git a/ydb/library/yql/tests/postgresql/cases/union.err b/ydb/library/yql/tests/postgresql/cases/union.err index 9fccd88351e9..df1a8e64fe7b 100644 --- a/ydb/library/yql/tests/postgresql/cases/union.err +++ b/ydb/library/yql/tests/postgresql/cases/union.err @@ -5,5 +5,5 @@ -- Simple UNION constructs SELECT 1 AS two UNION SELECT 2 ORDER BY 1; -NYql::TAggregateExpander::ExpandAggApply(TIntrusivePtr > const&) at /-S/ydb/library/yql/core/yql_aggregate_expander.cpp:119:51 +NYql::TAggregateExpander::ExpandAggApply(TIntrusivePtr > const&) at /-S/ydbwork/ydb/ydb/library/yql/core/yql_aggregate_expander.cpp:119:51 ?? at ??:0:0 diff --git a/ydb/library/yql/tests/postgresql/cases/window.err b/ydb/library/yql/tests/postgresql/cases/window.err index 81c5dff5806d..e785c75f109d 100644 --- a/ydb/library/yql/tests/postgresql/cases/window.err +++ b/ydb/library/yql/tests/postgresql/cases/window.err @@ -25,25 +25,9 @@ INSERT INTO empsalary VALUES SELECT depname, empno, salary, sum(salary) OVER (PARTITION BY depname) FROM empsalary ORDER BY depname, salary; --stdin-:
: Error: Type annotation - - -stdin-:
:1:1: Error: At function: AssumeColumnOrder, At function: Sort, At function: PgReplaceUnknown, At function: OrderedMap, At function: CalcOverWindow - SELECT depname, empno, salary, sum(salary) OVER (PARTITION BY depname) FROM empsalary ORDER BY depname, salary; - ^ - -stdin-:
:1:1: Error: Expected hashable and equatable type for key column: _yql_partition_key_0_0, but got: pgvarchar - SELECT depname, empno, salary, sum(salary) OVER (PARTITION BY depname) FROM empsalary ORDER BY depname, salary; - ^ SELECT depname, empno, salary, rank() OVER (PARTITION BY depname ORDER BY salary) FROM empsalary; --stdin-:
: Error: Type annotation - - -stdin-:
:1:1: Error: At function: AssumeColumnOrder, At function: PgReplaceUnknown, At function: OrderedMap, At function: CalcOverWindow - SELECT depname, empno, salary, rank() OVER (PARTITION BY depname ORDER BY salary) FROM empsalary; - ^ - -stdin-:
:1:1: Error: Expected hashable and equatable type for key column: _yql_partition_key_0_0, but got: pgvarchar - SELECT depname, empno, salary, rank() OVER (PARTITION BY depname ORDER BY salary) FROM empsalary; - ^ -- with GROUP BY SELECT four, ten, SUM(SUM(four)) OVER (PARTITION BY four), AVG(ten) FROM tenk1 @@ -56,14 +40,6 @@ GROUP BY four, ten ORDER BY four, ten; SELECT depname, empno, salary, sum(salary) OVER w FROM empsalary WINDOW w AS (PARTITION BY depname); --stdin-:
: Error: Type annotation - - -stdin-:
:1:1: Error: At function: AssumeColumnOrder, At function: PgReplaceUnknown, At function: OrderedMap, At function: CalcOverWindow - SELECT depname, empno, salary, sum(salary) OVER w FROM empsalary WINDOW w AS (PARTITION BY depname); - ^ - -stdin-:
:1:1: Error: Expected hashable and equatable type for key column: _yql_partition_key_0_0, but got: pgvarchar - SELECT depname, empno, salary, sum(salary) OVER w FROM empsalary WINDOW w AS (PARTITION BY depname); - ^ SELECT depname, empno, salary, rank() OVER w FROM empsalary WINDOW w AS (PARTITION BY depname ORDER BY salary) ORDER BY rank() OVER w; @@ -301,14 +277,6 @@ SELECT sum(salary), sum(sum(salary)) OVER (ORDER BY depname DESC) FROM empsalary GROUP BY depname; --stdin-:
: Error: Type annotation - - -stdin-:
:1:1: Error: At function: AssumeColumnOrder, At function: PgReplaceUnknown, At function: OrderedMap, At function: CalcOverWindow, At function: CalcOverWindow, At function: UnionAll, At function: Aggregate - -- more than one window with GROUP BY - ^ - -stdin-:
:1:1: Error: Expected hashable and equatable type for key column: _alias_empsalary.depname, but got: pgvarchar - -- more than one window with GROUP BY - ^ -- identical windows with different names SELECT sum(salary) OVER w1, count(*) OVER w2 @@ -335,14 +303,6 @@ SELECT count(*) OVER (PARTITION BY four) FROM (SELECT * FROM tenk1 WHERE FALSE)s -- mixture of agg/wfunc in the same window SELECT sum(salary) OVER w, rank() OVER w FROM empsalary WINDOW w AS (PARTITION BY depname ORDER BY salary DESC); --stdin-:
: Error: Type annotation - - -stdin-:
:1:1: Error: At function: AssumeColumnOrder, At function: PgReplaceUnknown, At function: OrderedMap, At function: CalcOverWindow - -- mixture of agg/wfunc in the same window - ^ - -stdin-:
:1:1: Error: Expected hashable and equatable type for key column: _yql_partition_key_0_0, but got: pgvarchar - -- mixture of agg/wfunc in the same window - ^ -- strict aggs SELECT empno, depname, salary, bonus, depadj, MIN(bonus) OVER (ORDER BY empno), MAX(depadj) OVER () FROM( @@ -2919,14 +2879,6 @@ SELECT * FROM FROM empsalary) emp WHERE first_emp = 1 OR last_emp = 1; --stdin-:
: Error: Type annotation - - -stdin-:
:1:1: Error: At function: PgSelect, At function: PgSetItem, At function: AssumeColumnOrder, At function: PgReplaceUnknown, At function: OrderedMap, At function: CalcOverWindow, At function: OrderedMap, At function: CalcOverWindow - SELECT * FROM - ^ - -stdin-:
:1:1: Error: Expected hashable and equatable type for key column: _yql_partition_key_0_0, but got: pgvarchar - SELECT * FROM - ^ -- cleanup DROP TABLE empsalary; @@ -3294,72 +3246,72 @@ SELECT i,AVG(v::bigint) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED SELECT i,AVG(v::int) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) FROM (VALUES(1,1),(2,2),(3,NULL),(4,NULL)) t(i,v); -VERIFY failed (2023-12-12T19:41:50.094283+0300): Value is not boxed +VERIFY failed (2024-01-12T02:19:37.139252+0300): Value is not boxed ydb/library/yql/public/udf/udf_value_inl.h:379 AsBoxed(): requirement IsBoxed() failed -BackTrace(void**, unsigned long)+29 (0x14087F6D) -FormatBackTrace(IOutputStream*)+32 (0x14088440) -PrintBackTrace()+17 (0x14088491) -NPrivate::InternalPanicImpl(int, char const*, char const*, int, int, int, TBasicStringBuf >, char const*, unsigned long)+995 (0x140CE743) -NPrivate::Panic(NPrivate::TStaticBuf const&, int, char const*, char const*, char const*, ...)+418 (0x140C18B2) -NYql::NUdf::TUnboxedValuePod::AsBoxed() const+218 (0x152AF12A) -NYql::PointerDatumFromPod(NYql::NUdf::TUnboxedValuePod const&)+29 (0x1B7A9E8D) -NYql::TPgResolvedCall::DoCalculate(NKikimr::NMiniKQL::TComputationContext&) const+771 (0x1B7E5C33) -NKikimr::NMiniKQL::TMutableComputationNode >::GetValue(NKikimr::NMiniKQL::TComputationContext&) const+420 (0x1B7E50D4) -??+0 (0x7F854D4FCC53) -??+0 (0x7F854D4FC47C) -??+0 (0x7F854D4FAE01) -??+0 (0x1FD82FD2) -NYql::NUdf::TBoxedValueAccessor::Fetch(NYql::NUdf::IBoxedValue&, NYql::NUdf::TUnboxedValue&)+244 (0x178E67A4) -NYql::NUdf::TUnboxedValuePod::Fetch(NYql::NUdf::TUnboxedValue&) const+228 (0x178E6624) -NKikimr::NMiniKQL::TForwardListValue::TIterator::Next(NYql::NUdf::TUnboxedValue&)+36 (0x17B99974) -NYql::NUdf::TBoxedValueAccessor::Next(NYql::NUdf::IBoxedValue&, NYql::NUdf::TUnboxedValue&)+244 (0x14EFDD24) -NYql::NUdf::TUnboxedValuePod::Next(NYql::NUdf::TUnboxedValue&) const+228 (0x14EFD9F4) -NYql::TYsonExecuteResOrPull::WriteValue(NYql::NUdf::TUnboxedValue const&, NKikimr::NMiniKQL::TType*)+436 (0x209F9A54) -NYql::NFile::TYtFileGateway::ExecuteResult(NYql::NFile::TSession&, NYson::TYsonWriter&, NYql::NNodes::TExprBase, NYql::TExprContext&, NYql::IYtGateway::TResOrPullOptions&&, TVector >, std::__y1::allocator > > > const&) const+2002 (0x1C34AE12) -NYql::NFile::TYtFileGateway::ResOrPull(TIntrusivePtr > const&, NYql::TExprContext&, NYql::IYtGateway::TResOrPullOptions&&)+1128 (0x1C31DEF8) -??+0 (0x1AB7ECFE) -??+0 (0x1AB82F89) -??+0 (0x1AB82EE8) -??+0 (0x1AB82E78) -??+0 (0x1AB82E28) -??+0 (0x1AB821BC) -std::__y1::__function::__value_func > const&, TIntrusivePtr >&, NYql::TExprContext&)> > > (TIntrusivePtr > const&, TIntrusivePtr >&, NYql::TExprContext&)>::operator()[abi:v15000](TIntrusivePtr > const&, TIntrusivePtr >&, NYql::TExprContext&) const+93 (0x1AB92C9D) -std::__y1::function > const&, TIntrusivePtr >&, NYql::TExprContext&)> > > (TIntrusivePtr > const&, TIntrusivePtr >&, NYql::TExprContext&)>::operator()(TIntrusivePtr > const&, TIntrusivePtr >&, NYql::TExprContext&) const+56 (0x1AB91FE8) -NYql::TExecTransformerBase::CallbackTransform(TIntrusivePtr > const&, TIntrusivePtr >&, NYql::TExprContext&)+811 (0x1AB91C6B) -NYql::TAsyncCallbackTransformer::DoTransform(TIntrusivePtr >, TIntrusivePtr >&, NYql::TExprContext&)+64 (0x1AB7FF50) -NYql::TGraphTransformerBase::Transform(TIntrusivePtr >, TIntrusivePtr >&, NYql::TExprContext&)+100 (0x153FF174) -??+0 (0x176F665C) -??+0 (0x176EA0A6) -NYql::TGraphTransformerBase::Transform(TIntrusivePtr >, TIntrusivePtr >&, NYql::TExprContext&)+100 (0x153FF174) -??+0 (0x1786F3EC) -??+0 (0x17861728) -??+0 (0x1786DCCC) -??+0 (0x178619D7) -??+0 (0x1786D98A) -??+0 (0x178619D7) -??+0 (0x17859F2E) -NYql::TGraphTransformerBase::Transform(TIntrusivePtr >, TIntrusivePtr >&, NYql::TExprContext&)+100 (0x153FF174) -??+0 (0x15403098) -??+0 (0x15402DC9) -??+0 (0x153FF7D5) -NYql::TGraphTransformerBase::Transform(TIntrusivePtr >, TIntrusivePtr >&, NYql::TExprContext&)+100 (0x153FF174) -NYql::AsyncTransformStepImpl(NYql::IGraphTransformer&, TIntrusivePtr >&, NYql::TExprContext&, bool, bool, TBasicStringBuf > const&)+1222 (0x153FE226) -NYql::AsyncTransform(NYql::IGraphTransformer&, TIntrusivePtr >&, NYql::TExprContext&, bool)+127 (0x153FEAAF) -NYql::TProgram::AsyncTransformWithFallback(bool)+120 (0x16C17A78) -??+0 (0x16C3EC1E) -??+0 (0x16C3E797) -??+0 (0x16C3E665) -??+0 (0x16C3D140) -??+0 (0x16C3D007) -??+0 (0x16C15F8D) -NYql::TProgram::RunAsync(TBasicString > const&, IOutputStream*, IOutputStream*, IOutputStream*, bool)+3760 (0x16C15C20) -??+0 (0x16C12804) -NYql::TProgram::Run(TBasicString > const&, IOutputStream*, IOutputStream*, IOutputStream*, bool)+301 (0x16C14CCD) -Main(int, char**)+4924 (0x13F14D4C) -main+273 (0x13F16831) -??+0 (0x7F854D229D90) -__libc_start_main+128 (0x7F854D229E40) -??+0 (0x13EA0029) +BackTrace(void**, unsigned long)+29 (0x142C329D) +FormatBackTrace(IOutputStream*)+32 (0x142C3770) +PrintBackTrace()+17 (0x142C37C1) +NPrivate::InternalPanicImpl(int, char const*, char const*, int, int, int, TBasicStringBuf >, char const*, unsigned long)+995 (0x143260F3) +NPrivate::Panic(NPrivate::TStaticBuf const&, int, char const*, char const*, char const*, ...)+418 (0x14319332) +NYql::NUdf::TUnboxedValuePod::AsBoxed() const+218 (0x154F450A) +NYql::PointerDatumFromPod(NYql::NUdf::TUnboxedValuePod const&)+29 (0x1BB15C6D) +NYql::TPgResolvedCall::DoCalculate(NKikimr::NMiniKQL::TComputationContext&) const+771 (0x1BB577E3) +NKikimr::NMiniKQL::TMutableComputationNode >::GetValue(NKikimr::NMiniKQL::TComputationContext&) const+420 (0x1BB56C84) +??+0 (0x7F17843F0C53) +??+0 (0x7F17843F047C) +??+0 (0x7F17843EEE01) +??+0 (0x2011E4F2) +NYql::NUdf::TBoxedValueAccessor::Fetch(NYql::NUdf::IBoxedValue&, NYql::NUdf::TUnboxedValue&)+244 (0x17B82E34) +NYql::NUdf::TUnboxedValuePod::Fetch(NYql::NUdf::TUnboxedValue&) const+228 (0x17B82CB4) +NKikimr::NMiniKQL::TForwardListValue::TIterator::Next(NYql::NUdf::TUnboxedValue&)+36 (0x17E35FF4) +NYql::NUdf::TBoxedValueAccessor::Next(NYql::NUdf::IBoxedValue&, NYql::NUdf::TUnboxedValue&)+244 (0x151403B4) +NYql::NUdf::TUnboxedValuePod::Next(NYql::NUdf::TUnboxedValue&) const+228 (0x15140084) +NYql::TYsonExecuteResOrPull::WriteValue(NYql::NUdf::TUnboxedValue const&, NKikimr::NMiniKQL::TType*)+436 (0x20D5B834) +NYql::NFile::TYtFileGateway::ExecuteResult(NYql::NFile::TSession&, NYson::TYsonWriter&, NYql::NNodes::TExprBase, NYql::TExprContext&, NYql::IYtGateway::TResOrPullOptions&&, TVector >, std::__y1::allocator > > > const&) const+2002 (0x1C6C1D32) +NYql::NFile::TYtFileGateway::ResOrPull(TIntrusivePtr > const&, NYql::TExprContext&, NYql::IYtGateway::TResOrPullOptions&&)+1128 (0x1C694F58) +??+0 (0x1AEE8ACE) +??+0 (0x1AEECD59) +??+0 (0x1AEECCB8) +??+0 (0x1AEECC48) +??+0 (0x1AEECBF8) +??+0 (0x1AEEBF8C) +std::__y1::__function::__value_func > const&, TIntrusivePtr >&, NYql::TExprContext&)> > > (TIntrusivePtr > const&, TIntrusivePtr >&, NYql::TExprContext&)>::operator()[abi:v15000](TIntrusivePtr > const&, TIntrusivePtr >&, NYql::TExprContext&) const+93 (0x1AEFCA0D) +std::__y1::function > const&, TIntrusivePtr >&, NYql::TExprContext&)> > > (TIntrusivePtr > const&, TIntrusivePtr >&, NYql::TExprContext&)>::operator()(TIntrusivePtr > const&, TIntrusivePtr >&, NYql::TExprContext&) const+56 (0x1AEFBD58) +NYql::TExecTransformerBase::CallbackTransform(TIntrusivePtr > const&, TIntrusivePtr >&, NYql::TExprContext&)+811 (0x1AEFB9DB) +NYql::TAsyncCallbackTransformer::DoTransform(TIntrusivePtr >, TIntrusivePtr >&, NYql::TExprContext&)+64 (0x1AEE9D20) +NYql::TGraphTransformerBase::Transform(TIntrusivePtr >, TIntrusivePtr >&, NYql::TExprContext&)+100 (0x15645D84) +??+0 (0x1799623C) +??+0 (0x17989C86) +NYql::TGraphTransformerBase::Transform(TIntrusivePtr >, TIntrusivePtr >&, NYql::TExprContext&)+100 (0x15645D84) +??+0 (0x17B2220C) +??+0 (0x17B14548) +??+0 (0x17B20AEC) +??+0 (0x17B147F7) +??+0 (0x17B207AA) +??+0 (0x17B147F7) +??+0 (0x17B0CD4E) +NYql::TGraphTransformerBase::Transform(TIntrusivePtr >, TIntrusivePtr >&, NYql::TExprContext&)+100 (0x15645D84) +??+0 (0x15649CA8) +??+0 (0x156499D9) +??+0 (0x156463E5) +NYql::TGraphTransformerBase::Transform(TIntrusivePtr >, TIntrusivePtr >&, NYql::TExprContext&)+100 (0x15645D84) +NYql::AsyncTransformStepImpl(NYql::IGraphTransformer&, TIntrusivePtr >&, NYql::TExprContext&, bool, bool, TBasicStringBuf > const&)+1222 (0x15644E36) +NYql::AsyncTransform(NYql::IGraphTransformer&, TIntrusivePtr >&, NYql::TExprContext&, bool)+127 (0x156456BF) +NYql::TProgram::AsyncTransformWithFallback(bool)+120 (0x16EA37F8) +??+0 (0x16ECA8FE) +??+0 (0x16ECA477) +??+0 (0x16ECA345) +??+0 (0x16EC8E20) +??+0 (0x16EC8CE7) +??+0 (0x16EA1D0D) +NYql::TProgram::RunAsync(TBasicString > const&, IOutputStream*, IOutputStream*, IOutputStream*, bool)+3760 (0x16EA19A0) +??+0 (0x16E9E584) +NYql::TProgram::Run(TBasicString > const&, IOutputStream*, IOutputStream*, IOutputStream*, bool)+301 (0x16EA0A4D) +Main(int, char**)+5378 (0x1414F2E2) +main+273 (0x14150F11) +??+0 (0x7F1784029D90) +__libc_start_main+128 (0x7F1784029E40) +??+0 (0x140DA029) pthread_kill at ./nptl/./nptl/pthread_kill.c:43:17 ?? at ??:0:0 diff --git a/ydb/library/yql/tests/postgresql/cases/xml.err b/ydb/library/yql/tests/postgresql/cases/xml.err index 7ac69d22136c..0b33ab4cb111 100644 --- a/ydb/library/yql/tests/postgresql/cases/xml.err +++ b/ydb/library/yql/tests/postgresql/cases/xml.err @@ -1358,12 +1358,7 @@ SELECT xml_is_well_formed('abc'); -- error messages, we suppress the DETAIL in this test. \set VERBOSITY terse --stdin-:
: Error: Parse Sql - - -stdin-:
:11:1: Error: ERROR: syntax error at or near "\" - - \set VERBOSITY terse - ^ +Metacommand \set VERBOSITY terse is not supported SELECT xpath('/*', ''); diff --git a/ydb/library/yql/tests/postgresql/cases/xml.err.1 b/ydb/library/yql/tests/postgresql/cases/xml.err.1 index 7ac69d22136c..0b33ab4cb111 100644 --- a/ydb/library/yql/tests/postgresql/cases/xml.err.1 +++ b/ydb/library/yql/tests/postgresql/cases/xml.err.1 @@ -1358,12 +1358,7 @@ SELECT xml_is_well_formed('abc'); -- error messages, we suppress the DETAIL in this test. \set VERBOSITY terse --stdin-:
: Error: Parse Sql - - -stdin-:
:11:1: Error: ERROR: syntax error at or near "\" - - \set VERBOSITY terse - ^ +Metacommand \set VERBOSITY terse is not supported SELECT xpath('/*', ''); diff --git a/ydb/library/yql/tests/postgresql/cases/xml.err.2 b/ydb/library/yql/tests/postgresql/cases/xml.err.2 index 7ac69d22136c..0b33ab4cb111 100644 --- a/ydb/library/yql/tests/postgresql/cases/xml.err.2 +++ b/ydb/library/yql/tests/postgresql/cases/xml.err.2 @@ -1358,12 +1358,7 @@ SELECT xml_is_well_formed('abc'); -- error messages, we suppress the DETAIL in this test. \set VERBOSITY terse --stdin-:
: Error: Parse Sql - - -stdin-:
:11:1: Error: ERROR: syntax error at or near "\" - - \set VERBOSITY terse - ^ +Metacommand \set VERBOSITY terse is not supported SELECT xpath('/*', ''); diff --git a/ydb/public/lib/ydb_cli/common/format.cpp b/ydb/public/lib/ydb_cli/common/format.cpp index 08b76da9781d..dc8f37c59d8b 100644 --- a/ydb/public/lib/ydb_cli/common/format.cpp +++ b/ydb/public/lib/ydb_cli/common/format.cpp @@ -439,6 +439,22 @@ void TQueryPlanPrinter::PrintPrettyTableImpl(const NJson::TJsonValue& plan, TStr newRow.Column(2, std::move(nRows)); } + NColorizer::TColors colors = NColorizer::AutoColors(Cout); + TStringBuf color; + switch(offset.size() % 3) { + case 0: + color = colors.Red(); + break; + case 1: + color = colors.Green(); + break; + case 2: + color = colors.Blue(); + break; + default: + color = colors.Default(); + break; + } if (node.contains("Operators")) { for (const auto& op : node.at("Operators").GetArraySafe()) { @@ -458,26 +474,25 @@ void TQueryPlanPrinter::PrintPrettyTableImpl(const NJson::TJsonValue& plan, TStr TStringBuilder operation; if (info.empty()) { - operation << offset << " -> " << op.GetMapSafe().at("Name").GetString(); + operation << offset << color << " -> " << colors.Default() << op.GetMapSafe().at("Name").GetString(); } else { - operation << offset << " -> " << op.GetMapSafe().at("Name").GetString() + operation << offset << color << " -> " << colors.Default() << op.GetMapSafe().at("Name").GetString() << " (" << JoinStrings(info, ", ") << ")"; } - auto& row = table.AddRow(); - row.Column(0, std::move(operation)); + newRow.Column(0, std::move(operation)); if (AnalyzeMode) { - row.Column(3, std::move(eCost)); - row.Column(4, std::move(eRows)); + newRow.Column(3, std::move(eCost)); + newRow.Column(4, std::move(eRows)); } else { - row.Column(1, std::move(eCost)); - row.Column(2, std::move(eRows)); + newRow.Column(1, std::move(eCost)); + newRow.Column(2, std::move(eRows)); } } } else { TStringBuilder operation; - operation << offset << " -> " << node.at("Node Type").GetString(); + operation << offset << color << " -> " << colors.Default() << node.at("Node Type").GetString(); newRow.Column(0, std::move(operation)); } @@ -614,6 +629,8 @@ NJson::TJsonValue TQueryPlanPrinter::ReconstructQueryPlanRec(const NJson::TJsonV TVector planInputs; + auto opName = op.GetMapSafe().at("Name").GetStringSafe(); + for (auto opInput : op.GetMapSafe().at("Inputs").GetArraySafe()) { if (opInput.GetMapSafe().contains("ExternalPlanNodeId")) { auto inputPlanKey = opInput.GetMapSafe().at("ExternalPlanNodeId").GetIntegerSafe(); @@ -623,6 +640,10 @@ NJson::TJsonValue TQueryPlanPrinter::ReconstructQueryPlanRec(const NJson::TJsonV auto inputPlanId = opInput.GetMapSafe().at("InternalOperatorId").GetIntegerSafe(); planInputs.push_back( ReconstructQueryPlanRec(plan, inputPlanId, planIndex, precomputes, nodeCounter)); } + // temp hack + if (opName == "Filter") { + break; + } } if (op.GetMapSafe().contains("Inputs")) { @@ -642,7 +663,7 @@ NJson::TJsonValue TQueryPlanPrinter::ReconstructQueryPlanRec(const NJson::TJsonV } } - result["Node Type"] = op.GetMapSafe().at("Name").GetStringSafe(); + result["Node Type"] = opName; NJson::TJsonValue newOps; newOps.AppendValue(op); result["Operators"] = newOps; diff --git a/ydb/public/lib/ydb_cli/common/pretty_table.cpp b/ydb/public/lib/ydb_cli/common/pretty_table.cpp index 0dc07572ee6e..b32c8ae9d00f 100644 --- a/ydb/public/lib/ydb_cli/common/pretty_table.cpp +++ b/ydb/public/lib/ydb_cli/common/pretty_table.cpp @@ -15,6 +15,48 @@ TPrettyTable::TRow::TRow(size_t nColumns) { } +size_t TotalAnsiEscapeCodeLen(TStringBuf text) { + enum { + TEXT, + BEFORE_CODE, + IN_CODE, + } state = TEXT; + + size_t totalLen = 0; + size_t curLen = 0; + + for (auto it = text.begin(); it < text.end(); ++it) { + switch (state) { + case TEXT: + if (*it == '\033') { + state = BEFORE_CODE; + curLen = 1; + } + break; + case BEFORE_CODE: + if (*it == '[') { + state = IN_CODE; + curLen++; + } else { + state = TEXT; + } + break; + case IN_CODE: + if (*it == ';' || isdigit(*it)) { + curLen++; + } else { + if (*it == 'm') { + totalLen += curLen + 1; + } + state = TEXT; + } + break; + } + } + + return totalLen; +} + size_t TPrettyTable::TRow::ExtraBytes(TStringBuf data) const { // counter of previously uncounted bytes size_t extraBytes = 0; @@ -31,7 +73,9 @@ size_t TPrettyTable::TRow::ExtraBytes(TStringBuf data) const { extraBytes += n - 1; } } - + // update counter with len of color + extraBytes += TotalAnsiEscapeCodeLen(data); + return extraBytes; } @@ -70,8 +114,11 @@ bool TPrettyTable::TRow::PrintColumns(IOutputStream& o, const TVector& w for (const auto& line : column) { data = line; extraBytes = ExtraBytes(data); + if (data && l < lineNumber) { + data.Skip(extraBytes); + } while (data && l < lineNumber) { - data.Skip(width + extraBytes); + data.Skip(width); ++l; } }