diff --git a/ydb/core/kqp/ut/federated_query/generic/ch_recipe_ut_helpers.cpp b/ydb/core/kqp/ut/federated_query/generic/ch_recipe_ut_helpers.cpp index d9e7116d1268..7b426cbdd468 100644 --- a/ydb/core/kqp/ut/federated_query/generic/ch_recipe_ut_helpers.cpp +++ b/ydb/core/kqp/ut/federated_query/generic/ch_recipe_ut_helpers.cpp @@ -6,19 +6,19 @@ namespace NTestUtils { TString GetChHost() { - return GetEnv("RECIPE_CLICKHOUSE_HOST", "localhost"); + return "localhost"; } ui32 GetChPort() { - return FromString(GetEnv("RECIPE_CLICKHOUSE_NATIVE_PORT", "1234")); + return 19000; } TString GetChUser() { - return GetEnv("RECIPE_CLICKHOUSE_USER"); + return "user"; } TString GetChPassword() { - return GetEnv("RECIPE_CLICKHOUSE_PASSWORD"); + return "password"; } TString GetChDatabase() { @@ -32,7 +32,20 @@ namespace NTestUtils { .SetPort(GetChPort()) .SetUser(GetChUser()) .SetPassword(GetChPassword()); - return NClickHouse::TClient(opt); + + TInstant start = TInstant::Now(); + ui32 attempt = 0; + while ((TInstant::Now() - start).Seconds() < 60) { + attempt += 1; + try { + return NClickHouse::TClient(opt); + } catch (const TSystemError& e) { + Cerr << "Attempt " << attempt << ": " << e.what() << Endl; + Sleep(TDuration::MilliSeconds(100)); + } + } + + throw yexception() << "Failed to connect ClickHouse in " << attempt << " attempt(s)"; } } // namespace NTestUtils diff --git a/ydb/core/kqp/ut/federated_query/generic/connector_recipe_ut_helpers.cpp b/ydb/core/kqp/ut/federated_query/generic/connector_recipe_ut_helpers.cpp index 23d5768f16f5..9b1ea42d4be5 100644 --- a/ydb/core/kqp/ut/federated_query/generic/connector_recipe_ut_helpers.cpp +++ b/ydb/core/kqp/ut/federated_query/generic/connector_recipe_ut_helpers.cpp @@ -6,13 +6,11 @@ namespace NTestUtils { TString GetConnectorHost() { - return GetEnv("YDB_CONNECTOR_RECIPE_GRPC_HOST", "localhost"); + return "localhost"; } ui32 GetConnectorPort() { - const TString port = GetEnv("YDB_CONNECTOR_RECIPE_GRPC_PORT"); - UNIT_ASSERT_C(port, "No connector port specified"); - return FromString(port); + return 50051; } std::shared_ptr MakeKikimrRunnerWithConnector() { diff --git a/ydb/core/kqp/ut/federated_query/generic/docker-compose.yml b/ydb/core/kqp/ut/federated_query/generic/docker-compose.yml new file mode 100644 index 000000000000..2991e0e883a1 --- /dev/null +++ b/ydb/core/kqp/ut/federated_query/generic/docker-compose.yml @@ -0,0 +1,25 @@ +version: '3.4' +services: + postgresql: + image: postgres:15-bullseye@sha256:3411b9f2e5239cd7867f34fcf22fe964230f7d447a71d63c283e3593d3f84085 + environment: + POSTGRES_DB: db + POSTGRES_USER: user + POSTGRES_PASSWORD: password + ports: + - 15432:5432 + clickhouse: + image: clickhouse/clickhouse-server:23-alpine@sha256:b078c1cd294632afa2aeba3530e7ba2e568513da23304354f455a25fab575c06 + environment: + CLICKHOUSE_DB: db + CLICKHOUSE_USER: user + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 + CLICKHOUSE_PASSWORD: password + ports: + - 19000:9000 + - 18123:8123 + fq-connector-go: + image: ghcr.io/ydb-platform/fq-connector-go:v0.0.6-rc.8@sha256:74ebae0530d916c1842a7fddfbddc6c018763a0401f2f627a44e8829692fe41f + ports: + - 50051:50051 + network_mode: host diff --git a/ydb/core/kqp/ut/federated_query/generic/kqp_generic_plan_ut.cpp b/ydb/core/kqp/ut/federated_query/generic/kqp_generic_plan_ut.cpp index 39cdcad63a78..f32177a2f377 100644 --- a/ydb/core/kqp/ut/federated_query/generic/kqp_generic_plan_ut.cpp +++ b/ydb/core/kqp/ut/federated_query/generic/kqp_generic_plan_ut.cpp @@ -43,7 +43,7 @@ Y_UNIT_TEST_SUITE(KqpGenericPlanTest) { { const TString sql = fmt::format( R"sql( - CREATE OBJECT pg_password_obj (TYPE SECRET) WITH (value=""); + CREATE OBJECT pg_password_obj (TYPE SECRET) WITH (value="{pg_password}"); CREATE EXTERNAL DATA SOURCE pg_data_source WITH ( SOURCE_TYPE="PostgreSQL", LOCATION="{pg_host}:{pg_port}", @@ -58,6 +58,7 @@ Y_UNIT_TEST_SUITE(KqpGenericPlanTest) { "pg_host"_a = GetPgHost(), "pg_port"_a = GetPgPort(), "pg_user"_a = GetPgUser(), + "pg_password"_a = GetPgPassword(), "pg_database"_a = GetPgDatabase()); auto result = session.ExecuteSchemeQuery(sql).GetValueSync(); UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); diff --git a/ydb/core/kqp/ut/federated_query/generic/kqp_generic_provider_join_ut.cpp b/ydb/core/kqp/ut/federated_query/generic/kqp_generic_provider_join_ut.cpp index de9679889836..63c62309b1c1 100644 --- a/ydb/core/kqp/ut/federated_query/generic/kqp_generic_provider_join_ut.cpp +++ b/ydb/core/kqp/ut/federated_query/generic/kqp_generic_provider_join_ut.cpp @@ -71,7 +71,7 @@ Y_UNIT_TEST_SUITE(FederatedQueryJoin) { { const TString sql = fmt::format( R"sql( - CREATE OBJECT pg_password_obj (TYPE SECRET) WITH (value=""); + CREATE OBJECT pg_password_obj (TYPE SECRET) WITH (value="{pg_password}"); CREATE EXTERNAL DATA SOURCE pg_data_source WITH ( SOURCE_TYPE="PostgreSQL", LOCATION="{pg_host}:{pg_port}", @@ -97,6 +97,7 @@ Y_UNIT_TEST_SUITE(FederatedQueryJoin) { "pg_host"_a = GetPgHost(), "pg_port"_a = GetPgPort(), "pg_user"_a = GetPgUser(), + "pg_password"_a = GetPgPassword(), "pg_database"_a = GetPgDatabase(), "ch_host"_a = GetChHost(), "ch_port"_a = GetChPort(), diff --git a/ydb/core/kqp/ut/federated_query/generic/pg_recipe_ut_helpers.cpp b/ydb/core/kqp/ut/federated_query/generic/pg_recipe_ut_helpers.cpp index 54a158c7d721..12b7ee6103e3 100644 --- a/ydb/core/kqp/ut/federated_query/generic/pg_recipe_ut_helpers.cpp +++ b/ydb/core/kqp/ut/federated_query/generic/pg_recipe_ut_helpers.cpp @@ -10,31 +10,47 @@ using namespace fmt::literals; namespace NTestUtils { TString GetPgHost() { - return GetEnv("POSTGRES_RECIPE_HOST", "localhost"); + return "localhost"; } ui32 GetPgPort() { - const TString port = GetEnv("POSTGRES_RECIPE_PORT"); - UNIT_ASSERT_C(port, "No postgresql port specified"); - return FromString(port); + return 15432; } TString GetPgUser() { - return GetEnv("POSTGRES_RECIPE_USER"); + return "user"; } TString GetPgDatabase() { - return GetEnv("POSTGRES_RECIPE_DBNAME"); + return "db"; + } + + TString GetPgPassword() { + return "password"; } pqxx::connection CreatePostgresqlConnection() { const TString connectionString = fmt::format( - "host={host} port={port} dbname={database} user={user}", + "host={host} port={port} dbname={database} user={user} password={password}", "host"_a = GetPgHost(), "port"_a = GetPgPort(), "database"_a = GetPgDatabase(), - "user"_a = GetPgUser()); - return pqxx::connection{connectionString}; + "user"_a = GetPgUser(), + "password"_a = GetPgPassword()); + + TInstant start = TInstant::Now(); + ui32 attempt = 0; + while ((TInstant::Now() - start).Seconds() < 60) { + attempt += 1; + try { + return pqxx::connection{connectionString}; + } catch (const pqxx::broken_connection& e) { + Cerr << "Attempt " << attempt << ": " << e.what() << Endl; + Sleep(TDuration::MilliSeconds(100)); + } + } + + throw yexception() << "Failed to connect PostgreSQL in " << attempt << " attempt(s)"; } } // namespace NTestUtils diff --git a/ydb/core/kqp/ut/federated_query/generic/pg_recipe_ut_helpers.h b/ydb/core/kqp/ut/federated_query/generic/pg_recipe_ut_helpers.h index c8c7bacffc3d..0e46e1ac3dc0 100644 --- a/ydb/core/kqp/ut/federated_query/generic/pg_recipe_ut_helpers.h +++ b/ydb/core/kqp/ut/federated_query/generic/pg_recipe_ut_helpers.h @@ -9,6 +9,7 @@ namespace NTestUtils { ui32 GetPgPort(); TString GetPgUser(); TString GetPgDatabase(); + TString GetPgPassword(); pqxx::connection CreatePostgresqlConnection(); diff --git a/ydb/core/kqp/ut/federated_query/generic/ya.make b/ydb/core/kqp/ut/federated_query/generic/ya.make index fa8dcad27152..3c5d43c2aa38 100644 --- a/ydb/core/kqp/ut/federated_query/generic/ya.make +++ b/ydb/core/kqp/ut/federated_query/generic/ya.make @@ -1,6 +1,24 @@ UNITTEST_FOR(ydb/core/kqp) -FORK_SUBTESTS() +IF (AUTOCHECK) + # Split tests to chunks only when they're running on different machines with distbuild, + # otherwise this directive will slow down local test execution. + # Look through https://st.yandex-team.ru/DEVTOOLSSUPPORT-39642 for more information. + FORK_SUBTESTS() + + # TAG and REQUIREMENTS are copied from: https://docs.yandex-team.ru/devtools/test/environment#docker-compose + TAG( + ya:external + ya:force_sandbox + ya:fat + ) + + REQUIREMENTS( + container:4467981730 + cpu:all + dns:dns64 + ) +ENDIF() SRCS( ch_recipe_ut_helpers.cpp @@ -20,9 +38,7 @@ PEERDIR( ydb/library/yql/sql/pg_dummy ) -INCLUDE(${ARCADIA_ROOT}/library/recipes/clickhouse/recipe.inc) -INCLUDE(${ARCADIA_ROOT}/library/recipes/postgresql/recipe.inc) -INCLUDE(${ARCADIA_ROOT}/ydb/library/yql/providers/generic/connector/recipe/recipe.inc) +INCLUDE(${ARCADIA_ROOT}/library/recipes/docker_compose/recipe.inc) YQL_LAST_ABI_VERSION()